commit 1b279d4d103b149c9e3beff6fc3b2fe69f50494e Author: wehub-resource-sync Date: Mon Jul 13 12:24:42 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..74e8438 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,16 @@ +[target.'cfg(target_os = "macos")'] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[target.aarch64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +# Android/Termux: no hardcoded linker so native Termux builds use the system cc. +# For CI cross-compilation the linker is set via CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER env var. +[target.aarch64-linux-android] +rustflags = ["-C", "link-args=-rdynamic"] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..3813ba8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,58 @@ +name: Bug report +description: Report a crash, hang, or incorrect behavior in fff (any frontend — nvim plugin, Node/Bun SDK, MCP server, C SDK). +title: "[Bug]: " +labels: ["bug"] +body: + - type: dropdown + id: frontend + attributes: + label: Which fff frontend? + options: + - Neovim plugin (fff.nvim) + - MCP server (fff-mcp) + - Node SDK (@ff-labs/fff-node) + - Bun SDK + - C SDK (libfff) + - Other / multiple + validations: + required: true + + - type: textarea + id: logs + attributes: + label: has logs + description: | + Attach your fff log file — the single most useful thing for debugging. + + fff writes a fresh log file on every process startup, named `fff++.log`, and keeps the last 20. Find the file matching your crashed/buggy run and drag-and-drop it here (or paste its contents). + + Where the log files live: + + | Frontend | Linux / macOS | Windows | + |---|---|---| + | Neovim plugin | `~/.local/state/nvim/log/fff+*.log` | `%LOCALAPPDATA%\nvim-data\log\fff+*.log` | + | MCP server (`fff-mcp`) | `~/.cache/fff_mcp+*.log` (override with `--log-file`) | `%LOCALAPPDATA%\fff_mcp+*.log` | + | Node / Bun SDK | path you passed as `logFilePath` to `FileFinder.create({...})` | same | + | C SDK | path you passed as `log_file_path` in `FffCreateOptions` | same | + + Neovim users: run `:FFFOpenLog` to open the current session's log directly. + + Also some useful commands: + + ```sh + # Neovim plugin + ls -t ~/.local/state/nvim/log/fff+*.log | head -1 + + # MCP server + ls -t ~/.cache/fff_mcp+*.log | head -1 + ``` + validations: + required: false + + - type: textarea + id: body + attributes: + label: Description + description: Please provide as much helpful information as you can + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0d3dd56 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Discussion / question + url: https://github.com/dmtrKovalenko/fff/discussions + about: For usage questions, design discussion, or anything that's not a bug or feature request. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..caa70e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,34 @@ +name: Feature request +description: Suggest something new for fff +title: "[Suggestion]: " +labels: ["enhancement"] +body: + - type: dropdown + id: frontend + attributes: + label: Which fff frontend(s)? + multiple: true + options: + - Neovim plugin (fff.nvim) + - MCP server (fff-mcp) + - Node SDK (@ff-labs/fff-node) + - Bun SDK (@ff-labs/fff-bun) + - C lib (libfff) + - Core or Rust crate + validations: + required: true + + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: If you have an idea of the shape of the API, describe it here. + validations: + required: false diff --git a/.github/workflows/external-tests.yml b/.github/workflows/external-tests.yml new file mode 100644 index 0000000..7c21232 --- /dev/null +++ b/.github/workflows/external-tests.yml @@ -0,0 +1,217 @@ +name: e2e Tests + +on: + push: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + pull_request: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + +env: + CARGO_TERM_COLOR: always + MACOSX_DEPLOYMENT_TARGET: "13" + # Force Node 24 for all JS-based actions to avoid the libuv + # process_title assertion crash on Windows (known Node 20 bug). + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + lua-tests: + name: e2e (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + # e2e tests could be flaky on CI so we do not block release creation if they failed + continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + - os: macos-latest + - os: windows-latest + target: x86_64-pc-windows-msvc + steps: + - uses: actions/checkout@v5 + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v6 + - name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1.15.4 + with: + cache: true + cache-on-failure: false + cache-key: "v2-lua-e2e" + rustflags: "" + target: ${{ matrix.target || '' }} + + - name: Build Rust binary (Windows) + if: matrix.target + run: cargo build --release --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob + + - name: Copy binary to target/release (Windows) + if: matrix.target + shell: bash + run: | + cp target/${{ matrix.target }}/release/fff_nvim.dll target/release/fff_nvim.dll + + - name: Verify Windows DLL has no unexpected dependencies + if: matrix.target + shell: pwsh + run: | + # Find dumpbin via vswhere (always available on GitHub Actions Windows runners) + $vsPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath + $dumpbin = Get-ChildItem "$vsPath" -Recurse -Filter "dumpbin.exe" | Select-Object -First 1 + if (-not $dumpbin) { Write-Error "dumpbin.exe not found"; exit 1 } + + $deps = & $dumpbin.FullName /DEPENDENTS target\release\fff_nvim.dll | Out-String + Write-Host $deps + # zlob must be statically linked - fail if zlob.dll appears as a dependency + if ($deps -match 'zlob\.dll') { + Write-Error "fff_nvim.dll has unexpected dynamic dependency on zlob.dll - zlob should be statically linked" + exit 1 + } + + - name: Build Rust binary + if: ${{ !matrix.target }} + run: cargo build --release -p fff-nvim --no-default-features --features zlob + + - name: Install Neovim + uses: rhysd/action-setup-vim@v1 + with: + neovim: true + version: v0.10.4 + + - name: Clone plenary.nvim + shell: bash + run: git clone --depth 1 https://github.com/nvim-lua/plenary.nvim ../plenary.nvim + + - name: Run Lua tests + shell: bash + run: make test-lua + + - name: Dump fff trace log on failure + if: failure() + shell: bash + run: | + # init_tracing writes session files named fff-test++.log + found=0 + for f in fff-test*.log; do + [ -f "$f" ] || continue + found=1 + echo "=== $f ===" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "(no log file produced)" + fi + + - name: Run version resolution tests + shell: bash + run: make test-version + + - name: Run non windows tests + shell: bash + if: ${{ matrix.os != 'windows-latest' }} + run: | + make test-bun + make test-c-api + + - name: Verify bun --compile + shell: bash + run: make test-bun-compile + + - name: Install Node.js + if: ${{ matrix.os != 'ubuntu-latest' }} + uses: actions/setup-node@v6 + with: + node-version: "25" + + - name: Install node dependencies + shell: bash + run: cd packages/fff-node && npm install + + - name: Run node tests + shell: bash + run: make test-node + + # Regression for https://github.com/dmtrKovalenko/fff/issues/480: build & + # run @ff-labs/fff-node end-to-end on real Alpine Linux (musl). Forces + # findBinary() through the npm-package resolver so detectLinuxLibc() + # actually runs. + alpine-musl: + name: e2e (alpine-musl) + runs-on: ubuntu-latest + container: node:22-alpine + continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }} + defaults: + run: + shell: sh + steps: + - name: Install build deps + run: apk add --no-cache git rust cargo musl-dev + + - uses: actions/checkout@v5 + + # libgit2 refuses repos owned by a different user; checkout in a + # container can land at a uid mismatch, so opt every dir in. + - name: Mark workspace safe for git + run: git config --global --add safe.directory '*' + + - name: Sanity check libc is musl + run: | + if ! ldd --version 2>&1 | grep -qi musl; then + echo "FAIL: container is not running musl libc" + exit 1 + fi + + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: alpine-musl-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + alpine-musl-cargo- + + - name: Build libfff_c (musl) + run: cargo build --release -p fff-c + + - name: Install workspace npm deps + run: npm install --no-package-lock + + # Upstream @yuuang/ffi-rs-linux-x64-musl ships with libc:"glibc" in + # its package.json (a publishing bug in ffi-rs), so npm filters it + # out. Force-install it so the FFI runtime is present on Alpine. + - name: Install ffi-rs musl runtime + run: | + FFI_RS_VERSION=$(node -p "require('ffi-rs/package.json').version") + npm install --no-package-lock --no-save --force \ + "@yuuang/ffi-rs-linux-x64-musl@${FFI_RS_VERSION}" + + # Stage the freshly built libfff_c.so as the platform npm package + # so findBinary() resolves through the @ff-labs/fff-bin-* path — + # this is what exercises detectLinuxLibc(). + - name: Stage musl bin package + run: | + PKG_DIR=node_modules/@ff-labs/fff-bin-linux-x64-musl + mkdir -p "$PKG_DIR" + cp target/release/libfff_c.so "$PKG_DIR/libfff_c.so" + cat >"$PKG_DIR/package.json" <<'JSON' + { "name": "@ff-labs/fff-bin-linux-x64-musl", "version": "0.0.0" } + JSON + + - name: Build fff-node + working-directory: packages/fff-node + run: npm run build + + - name: Run fff-node e2e suite + working-directory: packages/fff-node + run: node test/e2e.mjs diff --git a/.github/workflows/lua.yml b/.github/workflows/lua.yml new file mode 100644 index 0000000..5455d13 --- /dev/null +++ b/.github/workflows/lua.yml @@ -0,0 +1,56 @@ +name: Lua CI + +on: + push: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + pull_request: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + +jobs: + lua-ls: + name: lua-language-server type check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Neovim + run: | + curl -L https://github.com/neovim/neovim/releases/download/v0.11.5/nvim-linux-x86_64.tar.gz -o /opt/nvim.tar.gz + mkdir /opt/nvim + tar xzf /opt/nvim.tar.gz -C /opt/nvim + mv /opt/nvim/nvim-linux-x86_64/* /opt/nvim + echo "/opt/nvim/bin" >> $GITHUB_PATH + + - name: Install lua-language-server + run: | + curl -L "https://github.com/LuaLS/lua-language-server/releases/download/3.17.1/lua-language-server-3.17.1-linux-x64.tar.gz" -o /opt/lls.tar.gz + mkdir /opt/lls + tar -xzf /opt/lls.tar.gz -C /opt/lls + echo "/opt/lls/bin" >> $GITHUB_PATH + + - name: Clone snacks.nvim + run: git clone --depth=1 https://github.com/folke/snacks.nvim /opt/snacks.nvim + + - name: Run lua-language-server + run: lua-language-server --configpath .luarc.ci.json --check=. + + luacheck: + name: luacheck lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install luacheck + run: | + sudo apt-get update -qq + sudo apt-get install -y luarocks + sudo luarocks install luacheck + + - name: Run luacheck + run: luacheck lua/ diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 0000000..155eeca --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,37 @@ +name: Nix CI + +on: + push: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + pull_request: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + +jobs: + check: + runs-on: ubuntu-22.04 + permissions: + id-token: "write" + contents: "read" + steps: + - uses: actions/checkout@v5 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: DeterminateSystems/flake-checker-action@main + + - name: Run `nix flake check` + run: nix flake check + + - name: Run `nix build` + run: nix build + + - name: Run `nix build .#fff-nvim` + run: nix build .#fff-nvim + + - name: Run `nix run .#release` + run: nix run .#release diff --git a/.github/workflows/panvimdoc.yaml b/.github/workflows/panvimdoc.yaml new file mode 100644 index 0000000..d8d63a9 --- /dev/null +++ b/.github/workflows/panvimdoc.yaml @@ -0,0 +1,85 @@ +on: + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +name: docs + +jobs: + docs: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + with: + ref: main + fetch-depth: 2 + + - name: Extract Neovim section from README.md + run: | + awk ' + /^
/ { capture=1; next } + capture && /^<\/details>/ { capture=0; exit } + capture && /^$/ { next } + capture && /^<\/summary>$/ { next } + capture && /

.*<\/h2>/ { + gsub(/<\/?h2>/, "") + sub(/^[[:space:]]+/, "") + print "# " $0 + print "" + print "The best file search picker for Neovim. Frecency-ranked, typo-resistant, git-award, very fast." + print "" + next + } + capture { print } + ' README.md > .panvimdoc-input.md + test -s .panvimdoc-input.md + + - name: panvimdoc + uses: kdheepak/panvimdoc@main + with: + vimdoc: fff.nvim + pandoc: .panvimdoc-input.md + version: "Neovim >= 0.10.0" + demojify: true + treesitter: true + + - name: Cleanup intermediate file + run: rm -f .panvimdoc-input.md + + # panvimdoc stamps "Last change: " every run, so a daily cron always + # produces a one-line diff. Skip the PR unless a non-date line changed. + - name: Detect real doc changes + id: docdiff + run: | + if git diff --quiet -I 'Last change:' -- doc/fff.nvim.txt; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Create pull request + id: cpr + if: steps.docdiff.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v7 + with: + branch: bot/regenerate-vimdoc + token: ${{ secrets.GUSTAV_PAT }} + delete-branch: true + title: "chore: regenerate Neovim vimdoc" + commit-message: | + chore: regenerate Neovim vimdoc + + Co-authored-by: Dmitriy Kovalenko + author: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>" + committer: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>" + body: Automated vimdoc regeneration from README.md, scribed by Gustav. + add-paths: doc/fff.nvim.txt + + - name: Enable auto-merge + if: steps.cpr.outputs.pull-request-number + env: + GH_TOKEN: ${{ secrets.GUSTAV_PAT }} + run: gh pr merge --auto --squash "${{ steps.cpr.outputs.pull-request-number }}" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..43426ef --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,48 @@ +name: Python CI + +on: + push: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + pull_request: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + +env: + CARGO_TERM_COLOR: always + MACOSX_DEPLOYMENT_TARGET: "13.0" + +jobs: + test: + name: Python bindings (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.11.14" + enable-cache: true + + - name: Build and test Python bindings + working-directory: packages/fff-python + shell: bash + run: | + uv sync --all-extras + uv run maturin develop --release + uv run pytest -v diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..519f48b --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,757 @@ +name: Build & Publish + +on: + push: + branches: [main, fix/use-trusted-publishing] + tags: + - "v*" + pull_request: + workflow_dispatch: + inputs: + publish_pypi: + description: "Manually build and publish Python wheels to PyPI" + required: false + default: false + type: boolean + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + build-nvim: + name: Build Neovim ${{ matrix.target }} + runs-on: ${{ matrix.os }} + permissions: + contents: read + id-token: write + strategy: + matrix: + include: + ## Linux builds (using cargo-zigbuild) + # Glibc 2.31 (Ubuntu 20.04, Debian 11, RHEL 9). + # Rust 1.91+ requires glibc >= 2.31 for std::sys::random::getrandom, + # copy_file_range, and statx; earlier targets (2.17) no longer link. + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + zigbuild_target: x86_64-unknown-linux-gnu.2.31 + artifact_name: target/x86_64-unknown-linux-gnu/ci/libfff_nvim.so + ext: so + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + zigbuild_target: aarch64-unknown-linux-gnu.2.31 + artifact_name: target/aarch64-unknown-linux-gnu/ci/libfff_nvim.so + ext: so + # Musl (statically linked) + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + artifact_name: target/x86_64-unknown-linux-musl/ci/libfff_nvim.so + ext: so + - os: ubuntu-latest + target: aarch64-unknown-linux-musl + artifact_name: target/aarch64-unknown-linux-musl/ci/libfff_nvim.so + ext: so + + ## Android (Termux) + - os: ubuntu-latest + target: aarch64-linux-android + artifact_name: target/aarch64-linux-android/ci/libfff_nvim.so + ext: so + + ## macOS builds + - os: macos-latest + target: x86_64-apple-darwin + artifact_name: target/x86_64-apple-darwin/ci/libfff_nvim.dylib + ext: dylib + - os: macos-latest + target: aarch64-apple-darwin + artifact_name: target/aarch64-apple-darwin/ci/libfff_nvim.dylib + ext: dylib + + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_name: target/x86_64-pc-windows-msvc/ci/fff_nvim.dll + ext: dll + - os: windows-latest + target: aarch64-pc-windows-msvc + artifact_name: target/aarch64-pc-windows-msvc/ci/fff_nvim.dll + ext: dll + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install Rust + run: rustup target add ${{ matrix.target }} + + - name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install cargo-zigbuild + if: contains(matrix.os, 'ubuntu') + run: cargo install cargo-zigbuild + + - name: Build for Linux + if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android') + run: | + cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for Android (Termux) + if: contains(matrix.target, 'android') + run: | + NDK_BIN="$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin" + + # NDK clang for C deps (libgit2, lmdb, blake3) that need Bionic sysroot headers + export CC_aarch64_linux_android="$NDK_BIN/aarch64-linux-android24-clang" + export CXX_aarch64_linux_android="$NDK_BIN/aarch64-linux-android24-clang++" + export AR_aarch64_linux_android="$NDK_BIN/llvm-ar" + export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang" + + cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for macOS + if: contains(matrix.os, 'macos') + run: | + MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}" + + - name: Ad-hoc sign macOS binary + if: contains(matrix.os, 'macos') + run: codesign --force --sign - "${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for Windows + if: contains(matrix.os, 'windows') + shell: bash + run: | + cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: nvim-${{ matrix.target }} + path: ${{ matrix.target }}.* + + build-c: + name: Build C FFI ${{ matrix.target }} + runs-on: ${{ matrix.os }} + permissions: + contents: read + strategy: + matrix: + include: + ## Linux builds + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + zigbuild_target: x86_64-unknown-linux-gnu.2.31 + artifact_name: target/x86_64-unknown-linux-gnu/ci/libfff_c.so + npm_package: fff-bin-linux-x64-gnu + lib_filename: libfff_c.so + ext: so + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + zigbuild_target: aarch64-unknown-linux-gnu.2.31 + artifact_name: target/aarch64-unknown-linux-gnu/ci/libfff_c.so + npm_package: fff-bin-linux-arm64-gnu + lib_filename: libfff_c.so + ext: so + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + artifact_name: target/x86_64-unknown-linux-musl/ci/libfff_c.so + npm_package: fff-bin-linux-x64-musl + lib_filename: libfff_c.so + ext: so + - os: ubuntu-latest + target: aarch64-unknown-linux-musl + artifact_name: target/aarch64-unknown-linux-musl/ci/libfff_c.so + npm_package: fff-bin-linux-arm64-musl + lib_filename: libfff_c.so + ext: so + + ## Android (Termux) + - os: ubuntu-latest + target: aarch64-linux-android + artifact_name: target/aarch64-linux-android/ci/libfff_c.so + lib_filename: libfff_c.so + ext: so + + ## macOS builds + - os: macos-latest + target: x86_64-apple-darwin + artifact_name: target/x86_64-apple-darwin/ci/libfff_c.dylib + npm_package: fff-bin-darwin-x64 + lib_filename: libfff_c.dylib + ext: dylib + - os: macos-latest + target: aarch64-apple-darwin + artifact_name: target/aarch64-apple-darwin/ci/libfff_c.dylib + npm_package: fff-bin-darwin-arm64 + lib_filename: libfff_c.dylib + ext: dylib + + ## Windows builds + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_name: target/x86_64-pc-windows-msvc/ci/fff_c.dll + npm_package: fff-bin-win32-x64 + lib_filename: fff_c.dll + ext: dll + - os: windows-latest + target: aarch64-pc-windows-msvc + artifact_name: target/aarch64-pc-windows-msvc/ci/fff_c.dll + npm_package: fff-bin-win32-arm64 + lib_filename: fff_c.dll + ext: dll + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install Rust + run: rustup target add ${{ matrix.target }} + + - name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install cargo-zigbuild + if: contains(matrix.os, 'ubuntu') + run: cargo install cargo-zigbuild + + - name: Build for Linux + if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android') + run: | + cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for Android (Termux) + if: contains(matrix.target, 'android') + run: | + NDK_BIN="$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin" + + export CC_aarch64_linux_android="$NDK_BIN/aarch64-linux-android24-clang" + export CXX_aarch64_linux_android="$NDK_BIN/aarch64-linux-android24-clang++" + export AR_aarch64_linux_android="$NDK_BIN/llvm-ar" + export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang" + + cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for macOS + if: contains(matrix.os, 'macos') + run: | + MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}" + + - name: Ad-hoc sign macOS binary + if: contains(matrix.os, 'macos') + run: codesign --force --sign - "c-lib-${{ matrix.target }}.${{ matrix.ext }}" + + - name: Build for Windows + if: contains(matrix.os, 'windows') + shell: bash + run: | + cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob + mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}" + + - name: Prepare npm package + if: "!contains(matrix.target, 'android')" + shell: bash + run: | + # Copy the built binary into the platform npm package directory + cp "c-lib-${{ matrix.target }}.${{ matrix.ext }}" "packages/${{ matrix.npm_package }}/${{ matrix.lib_filename }}" + + - name: Upload C library artifact + uses: actions/upload-artifact@v4 + with: + name: c-lib-${{ matrix.target }} + path: c-lib-${{ matrix.target }}.* + + - name: Upload npm package artifact + if: "!contains(matrix.target, 'android')" + uses: actions/upload-artifact@v4 + with: + name: npm-${{ matrix.npm_package }} + path: packages/${{ matrix.npm_package }}/ + + build-mcp: + name: Build MCP ${{ matrix.target }} + runs-on: ${{ matrix.os }} + permissions: + contents: read + strategy: + matrix: + include: + ## Linux builds (using cargo-zigbuild) + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + zigbuild_target: x86_64-unknown-linux-gnu.2.31 + artifact_name: target/x86_64-unknown-linux-gnu/ci/fff-mcp + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + zigbuild_target: aarch64-unknown-linux-gnu.2.31 + artifact_name: target/aarch64-unknown-linux-gnu/ci/fff-mcp + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + artifact_name: target/x86_64-unknown-linux-musl/ci/fff-mcp + - os: ubuntu-latest + target: aarch64-unknown-linux-musl + artifact_name: target/aarch64-unknown-linux-musl/ci/fff-mcp + + ## macOS builds + - os: macos-latest + target: x86_64-apple-darwin + artifact_name: target/x86_64-apple-darwin/ci/fff-mcp + - os: macos-latest + target: aarch64-apple-darwin + artifact_name: target/aarch64-apple-darwin/ci/fff-mcp + + ## Windows builds + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_name: target/x86_64-pc-windows-msvc/ci/fff-mcp.exe + - os: windows-latest + target: aarch64-pc-windows-msvc + artifact_name: target/aarch64-pc-windows-msvc/ci/fff-mcp.exe + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install Rust + run: rustup target add ${{ matrix.target }} + + - name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install cargo-zigbuild + if: contains(matrix.os, 'ubuntu') + run: cargo install cargo-zigbuild + + - name: Build for Linux + if: contains(matrix.os, 'ubuntu') + run: | + cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --no-default-features --features zlob + cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}" + + - name: Build for macOS + if: contains(matrix.os, 'macos') + run: | + MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob + cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}" + + - name: Ad-hoc sign macOS binary + if: contains(matrix.os, 'macos') + run: codesign --force --sign - "fff-mcp-${{ matrix.target }}" + + - name: Build for Windows + if: contains(matrix.os, 'windows') + shell: bash + run: | + cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob + cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}.exe" + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: mcp-${{ matrix.target }} + path: fff-mcp-${{ matrix.target }}* + + build-python: + name: Build Python wheels ${{ matrix.target }} (${{ matrix.os }}) + # Wheels are release artifacts; PR validation uses the develop build in + # python.yml, so skip the cross-compile matrix on pull requests. + if: github.event_name != 'pull_request' + runs-on: ${{ matrix.os }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64 + container: "off" + - os: ubuntu-latest + target: aarch64 + container: "off" + - os: macos-latest + target: x86_64 + - os: macos-latest + target: aarch64 + - os: windows-latest + target: x86_64 + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install cargo-zigbuild + if: contains(matrix.os, 'ubuntu') + run: cargo install cargo-zigbuild + + - name: Install aarch64 cross compiler + if: matrix.target == 'aarch64' && contains(matrix.os, 'ubuntu') + run: | + sudo apt-get update -qq + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Build wheels + uses: PyO3/maturin-action@v1 + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc + CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ + AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar + with: + target: ${{ matrix.target }} + args: --release --out dist --no-default-features --features zlob + sccache: "true" + working-directory: packages/fff-python + container: ${{ matrix.container || '' }} + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: python-wheels-${{ matrix.os }}-${{ matrix.target }} + path: packages/fff-python/dist/ + + build-python-sdist: + name: Build Python sdist + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + working-directory: packages/fff-python + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: python-sdist + path: packages/fff-python/dist/ + + release: + name: Release + needs: [build-nvim, build-c, build-mcp, build-python, build-python-sdist] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v')) + permissions: + contents: write + steps: + - uses: actions/checkout@v5 + + - name: Install Lua + uses: leafo/gh-actions-lua@v12 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: ./binaries + + - name: Flatten and rename Neovim artifacts + working-directory: ./binaries + run: | + # Move nvim artifacts to root level with original naming + for dir in nvim-*/; do + target="${dir#nvim-}" + target="${target%/}" + for file in "$dir"*; do + if [ -f "$file" ]; then + filename=$(basename "$file") + mv "$file" "./$filename" + fi + done + rmdir "$dir" 2>/dev/null || true + done + + - name: Flatten C library artifacts + working-directory: ./binaries + run: | + # Move c-lib artifacts to root level + for dir in c-lib-*/; do + for file in "$dir"*; do + if [ -f "$file" ]; then + filename=$(basename "$file") + mv "$file" "./$filename" + fi + done + rmdir "$dir" 2>/dev/null || true + done + + - name: Flatten MCP artifacts + working-directory: ./binaries + run: | + for dir in mcp-*/; do + for file in "$dir"*; do + if [ -f "$file" ]; then + filename=$(basename "$file") + mv "$file" "./$filename" + fi + done + rmdir "$dir" 2>/dev/null || true + done + + - name: Move Python wheels to release directory + working-directory: ./binaries + run: | + mkdir -p python + for dir in python-wheels-*/ python-sdist/; do + [ -d "$dir" ] || continue + for file in "$dir"*; do + if [ -f "$file" ]; then + mv "$file" "python/$(basename "$file")" + fi + done + rmdir "$dir" 2>/dev/null || true + done + + - name: Remove npm package artifacts from release binaries + working-directory: ./binaries + run: | + rm -rf npm-* + + - name: Generate checksums + working-directory: ./binaries + run: | + ls -la + for file in * python/*; do + if [ -f "$file" ] && [[ ! "$file" == *.sha256 ]]; then + sha256sum "$file" > "${file}.sha256" + fi + done + + - name: Determine version + id: version + run: lua scripts/determine-version.lua + + # Nightlies publish to a permanent per-sha tag (release_tag == version) so + # pinned/stale installs always fetch the binary built for their own commit. + # The rolling `nightly` tag is also moved to HEAD for "give me latest" tooling. + - name: Move rolling nightly tag to current commit + if: steps.version.outputs.is_release != 'true' + run: | + git tag -f nightly "${{ github.sha }}" + git push -f origin refs/tags/nightly + + - name: Upload Release Assets + uses: softprops/action-gh-release@v2 + with: + name: "${{ steps.version.outputs.version }}" + tag_name: "${{ steps.version.outputs.release_tag }}" + token: ${{ github.token }} + files: | + ./binaries/* + ./binaries/python/* + draft: false + prerelease: ${{ steps.version.outputs.is_release != 'true' }} + generate_release_notes: ${{ steps.version.outputs.is_release == 'true' }} + body: | + ${{ steps.version.outputs.is_release == 'true' && format('Release {0}', steps.version.outputs.version) || format('Nightly release from commit: {0}', github.sha) }} + + npm packages, rust crates and python wheels are available under this version ${{ steps.version.outputs.version }} + + ## Neovim Plugin + - `{target}.so` / `.dylib` / `.dll` - Lua module for Neovim + + ## C FFI Library (for Bun/Node/Python) + - `c-lib-{target}.so` / `.dylib` / `.dll` - C FFI library + + ## MCP Server + - `fff-mcp-{target}` - MCP server binary + + ## Python Package + - `python/*.whl` / `python/*.tar.gz` - Python wheels and sdist + - Install from PyPI: `pip install fff-search` (when published) + + Update mcp via: + ```sh + curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash + ``` + + - name: Bump Homebrew formula (uses local checksums) + if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko' + run: make bump-homebrew-formula VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries + + - name: Pin SHAs in install-mcp.sh (uses local checksums) + if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko' + run: make bump-install-mcp-sh VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries + + - name: Commit formula + installer bump to main + # Uses the default GITHUB_TOKEN configured by actions/checkout above. + # Requires github-actions[bot] in the main branch-protection bypass list. + if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + # Workflow runs on detached HEAD at the v* tag — explicit target needed. + branch: main + commit_message: "chore: bump fff-mcp release artifacts to v${{ steps.version.outputs.version }}" + file_pattern: "Formula/fff-mcp.rb install-mcp.sh" + commit_user_name: github-actions[bot] + commit_user_email: 41898282+github-actions[bot]@users.noreply.github.com + + pypi-publish: + name: Publish Python wheels to PyPI + needs: [build-python, build-python-sdist] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v')) + environment: + name: pypi + url: https://pypi.org/p/fff-search + permissions: + contents: read + id-token: write + steps: + - name: Download Python wheels and sdist + uses: actions/download-artifact@v4 + with: + pattern: python-* + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + skip-existing: true + + crates-publish: + name: Publish Rust crates + needs: [build-nvim, build-c, build-mcp] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v')) + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v5 + - uses: rust-lang/crates-io-auth-action@v1 + id: auth + + - name: Install Lua + uses: leafo/gh-actions-lua@v12 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-edit + run: cargo install cargo-edit --force --locked + + - name: Determine version + id: version + run: lua scripts/determine-version.lua + + - name: Publish crates + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + run: make publish-crates V="${{ steps.version.outputs.version }}" + + npm-publish: + name: Publish npm packages + needs: [build-c] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v')) + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v5 + + - name: Install Lua + uses: leafo/gh-actions-lua@v12 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "25" + registry-url: "https://registry.npmjs.org" + + - name: Determine version + id: version + run: lua scripts/determine-version.lua + + - name: Download npm package artifacts + uses: actions/download-artifact@v4 + with: + pattern: npm-* + path: ./npm-packages + + - name: Publish platform packages + run: | + VERSION="${{ steps.version.outputs.version }}" + TAG="${{ steps.version.outputs.npm_tag }}" + + for pkg_dir in ./npm-packages/npm-*/; do + if [ -d "$pkg_dir" ]; then + pkg_name=$(node -p "require('${pkg_dir}package.json').name") + echo "Publishing ${pkg_name}@${VERSION} with tag ${TAG}..." + + make set-npm-version PKG="$pkg_dir" VERSION="$VERSION" + + cd "$pkg_dir" + npm publish --tag "$TAG" --access public --provenance + cd - + fi + done + + - name: Publish bun package + run: | + VERSION="${{ steps.version.outputs.version }}" + TAG="${{ steps.version.outputs.npm_tag }}" + + echo "Publishing @ff-labs/fff-bun@${VERSION} with tag ${TAG}..." + make set-npm-version PKG=packages/fff-bun VERSION="$VERSION" + + cd packages/fff-bun + npm publish --tag "$TAG" --access public --provenance + + - name: Publish Node.js package + run: | + VERSION="${{ steps.version.outputs.version }}" + TAG="${{ steps.version.outputs.npm_tag }}" + + echo "Publishing @ff-labs/fff-node@${VERSION} with tag ${TAG}..." + make set-npm-version PKG=packages/fff-node VERSION="$VERSION" + + cd packages/fff-node + npm install + npm run build + npm publish --tag "$TAG" --access public --provenance + + - name: Publish pi-fff package + run: | + VERSION="${{ steps.version.outputs.version }}" + TAG="${{ steps.version.outputs.npm_tag }}" + + echo "Publishing @ff-labs/pi-fff@${VERSION} with tag ${TAG}..." + make set-npm-version PKG=packages/pi-fff VERSION="$VERSION" + + cd packages/pi-fff + npm publish --tag "$TAG" --access public --provenance diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..1d60b0f --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,162 @@ +name: Rust CI + +on: + push: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + pull_request: + branches: [main] + paths-ignore: + - '**.md' + - 'doc/**' + +env: + CARGO_TERM_COLOR: always + # Ensure consistent macOS deployment target across all compiled objects + # (Rust, cc-compiled C code, and Zig-compiled zlob) to avoid linker warnings + MACOSX_DEPLOYMENT_TARGET: "13" + +jobs: + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + # Guard against deadlocks in the shared-picker / watcher teardown + # path: a stuck test would otherwise consume a full 6h CI slot. + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + + # Zig is required to compile zlob + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1.15.4 + with: + cache: true + cache-on-failure: true + cache-key: "v1-rust" + components: rustfmt, clippy + + - name: Run tests + run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim + + stress-test: + name: Fuzz Tests + runs-on: ${{ matrix.os }} + strategy: + # Keep going after one OS fails so we can see whether a bug + # reproduces everywhere or is platform-specific. + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + # Long-running; don't let a stuck watcher thread burn a full CI + # timeout. Two scenarios should finish well under this limit. + timeout-minutes: 20 + env: + FFF_STRESS_CASES: "5" + FFF_STRESS_MIN_OPS: "30" + FFF_STRESS_MAX_OPS: "60" + steps: + - uses: actions/checkout@v5 + + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1.15.4 + with: + cache: true + cache-on-failure: true + cache-key: "v1-rust-stress-${{ matrix.os }}" + components: rustfmt, clippy + + - name: Stress test seeded + shell: bash + run: make test-stress-seeded + + - name: Stress test random + shell: bash + run: make test-stress-random + + - name: Stress test regressions + shell: bash + run: make test-stress-regressions + + - name: Upload proptest regressions on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: proptest-regressions-${{ matrix.os }} + path: crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions + if-no-files-found: ignore + + build-i686: + name: Build i686-unknown-linux-gnu + runs-on: ubuntu-latest + # Verifies that fff-search compiles on 32-bit x86, where std::arch::x86_64 + # is unavailable. SIMD paths are disabled on this target; only the scalar + # fallback should build. See issue #656. + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + + - name: Install cross toolchain + run: | + sudo apt-get update + sudo apt-get install -y gcc-multilib g++-multilib + + - name: Install Rust (i686 target) + uses: actions-rust-lang/setup-rust-toolchain@v1.15.4 + with: + target: i686-unknown-linux-gnu + cache: true + cache-on-failure: true + cache-key: "v1-rust-i686" + + - name: Build fff-search for i686 + run: cargo build -p fff-search --target i686-unknown-linux-gnu + + fmt: + name: cargo fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: rustfmt + - name: Check formatting + run: cargo fmt -- --check + + clippy: + name: cargo clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + # Zig is required to compile zlob + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.16.0 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy + + - name: Run clippy + run: cargo clippy --no-default-features --features zlob -- -D warnings diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml new file mode 100644 index 0000000..b3b8d41 --- /dev/null +++ b/.github/workflows/spelling.yaml @@ -0,0 +1,25 @@ +name: Spelling + +permissions: + contents: read + +on: + push: + branches: + - main + pull_request: + +env: + CLICOLOR: 1 + +jobs: + spelling: + name: Spell Check with Typos + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Spell Check Repo + uses: crate-ci/typos@685eb3d55be2f85191e8c84acb9f44d7756f84ab # v1.29.4 diff --git a/.github/workflows/stylua.yaml b/.github/workflows/stylua.yaml new file mode 100644 index 0000000..fa7edb1 --- /dev/null +++ b/.github/workflows/stylua.yaml @@ -0,0 +1,37 @@ +name: Stylua + +permissions: + contents: read + +on: + push: + branches: + - main + paths: + - "**/*.lua" + - .stylua.toml + - .github/workflows/stylua.yaml + pull_request: + paths: + - "**/*.lua" + - .stylua.toml + - .github/workflows/stylua.yaml + +env: + CLICOLOR: 1 + +jobs: + stylua: + name: Check lua files using Stylua + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Stylua Check Repo + uses: JohnnyMorganz/stylua-action@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: latest + args: --color=always --check . diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d026b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,65 @@ +doc/tags +big-repo +target/ +.archive.lua +_*.lua +.lazy.lua +dual/ +result +.direnv +.devenv +.repro/ +.wrangler/ +*.so +*.dylib +# all the perf like utility files +*.data +node_modules/ +crates/fff-notify-debouncer-full/ +packages/fff-bun/glob-bench-bin + +dist/ +scripts/benchmark-results/ + +# Native binaries (downloaded at install) +*.dylib +*.so +*.dll +*.pdb + +# Instruments traces +*.trace/ + +# Test logs +fff-test.log + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.pyd +*.egg-info/ +*.egg +.eggs/ +build/ +*.whl +# Virtual environments +.venv/ +venv/ +env/ +ENV/ +# uv +# Testing / linting +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +# IDEs +.idea/ +.vscode/ +*.swp +*.swo +*~ diff --git a/.luacheckrc b/.luacheckrc new file mode 100644 index 0000000..706514b --- /dev/null +++ b/.luacheckrc @@ -0,0 +1,25 @@ +-- luacheck configuration for fff.nvim +-- https://luacheck.readthedocs.io/en/stable/config.html + +-- Neovim globals +globals = { "vim" } + +-- Standard library +std = "luajit" + +-- Ignore line length (handled by stylua) +max_line_length = false + +-- Ignore unused self argument in methods +self = false + +-- Files/directories to ignore +exclude_files = { + ".luarocks/", +} + +-- Warn about unused variables, but allow _ prefix convention +unused_args = true +ignore = { + "212", -- unused argument (too noisy for callback-heavy code) +} diff --git a/.luarc.ci.json b/.luarc.ci.json new file mode 100644 index 0000000..f16b379 --- /dev/null +++ b/.luarc.ci.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json", + "runtime": { + "version": "LuaJIT", + "pathStrict": true + }, + "workspace": { + "library": [ + "/opt/nvim/share/nvim/runtime/lua/vim/_meta", + "/opt/nvim/share/nvim/runtime/lua/vim/shared.lua", + "${3rd}/luv/library", + "${3rd}/busted/library", + "/opt/snacks.nvim/lua" + ], + "checkThirdParty": false + }, + "diagnostics": { + "severity": { + "undefined-global": "Error", + "undefined-field": "Warning", + "missing-return": "Warning", + "redundant-parameter": "Warning", + "param-type-mismatch": "Warning", + "assign-type-mismatch": "Warning", + "cast-type-mismatch": "Warning", + "deprecated": "Warning", + "undefined-doc-param": "Warning" + }, + "neededFileStatus": { + "undefined-global": "Any", + "undefined-field": "Any", + "missing-return": "Any", + "redundant-parameter": "Any", + "param-type-mismatch": "Any", + "assign-type-mismatch": "Any", + "cast-type-mismatch": "Any", + "deprecated": "Any", + "undefined-doc-param": "Any" + } + }, + "type": { + "checkTableShape": true + } +} diff --git a/.luarc.json b/.luarc.json new file mode 100644 index 0000000..6073601 --- /dev/null +++ b/.luarc.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json", + "runtime": { + "version": "LuaJIT" + }, + "workspace": { + "library": [ + "$VIMRUNTIME/lua", + "${3rd}/luv/library", + "${3rd}/busted/library" + ], + "checkThirdParty": false + }, + "diagnostics": { + "globals": ["vim"], + "severity": { + "undefined-global": "Error", + "undefined-field": "Warning", + "missing-return": "Warning", + "redundant-parameter": "Warning", + "param-type-mismatch": "Warning", + "assign-type-mismatch": "Warning", + "cast-type-mismatch": "Warning", + "deprecated": "Warning", + "undefined-doc-param": "Warning" + }, + "neededFileStatus": { + "undefined-global": "Any", + "undefined-field": "Any", + "missing-return": "Any", + "redundant-parameter": "Any", + "param-type-mismatch": "Any", + "assign-type-mismatch": "Any", + "cast-type-mismatch": "Any", + "deprecated": "Any", + "undefined-doc-param": "Any" + } + }, + "type": { + "checkTableShape": true + } +} diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..e698318 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "fff": { + "type": "stdio", + "command": "./target/release/fff-mcp", + "args": [] + } + } +} diff --git a/.nixignore b/.nixignore new file mode 100644 index 0000000..b6abdc8 --- /dev/null +++ b/.nixignore @@ -0,0 +1,4 @@ +empty_config.lua +benches/ +doc/ + diff --git a/.stylua.toml b/.stylua.toml new file mode 100644 index 0000000..6d75cf6 --- /dev/null +++ b/.stylua.toml @@ -0,0 +1,7 @@ +column_width = 120 +line_endings = "Unix" +indent_type = "Spaces" +indent_width = 2 +quote_style = "AutoPreferSingle" +call_parentheses = "Always" +collapse_simple_statement = "Always" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5da8e1e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,111 @@ +# To Clankers + +This repository contains **FFF.nvim (Fast File Finder)**, a high-performance file picker for Neovim inspired by blink.cmp's fuzzy matching technology. It's NOT a completion plugin, but rather a standalone file finder with advanced fuzzy search and frecency scoring. The project aims to be the drop-in replacement for telescope, fzf-lua, snacks.picker and similar plugins, focusing on speed, accuracy search and usability features. + +## Development Commands + +Always prefer Makefile commands listed to the cargo/bun/node if possible. + +### Building + +- `make build` - build everything + +### Testing and Development Tools + +This project does not have a traditional test suite. Testing is done through: + +- Create e2e local test file for Neovim: Load any Lua test file with `nvim -l ` +- Write inline rust unit tests for any functionality that is standalone and scoped within a single function + +### Code Quality + +- `make lint` - Rust linting and code analysis +- `make format` - Format all code +- `make test` - Run unit tests (limited coverage, primarily integration testing) + +When doing code make sure to REDUCE SIZE OF COMMENTS. This is very important. Every comment should be concise 1-2 liner maximum 4 lines if describes really extensive and unnatural concept. + +### Important coding rules + +- Do not add doc comments to the private structs and functions. +- Do not make public structs if something can be private + + +## Architecture + +Everything that is performance critical happens in rust world, everything that is neovim specific happens in the lua code. + +There are 3 main components: + +- Rust binary with the global file picker state containing index of all files +- Background thread with the file system watcher that updates the index in real time +- Lua UI layer that renders the picker, handles user input, and calls the rust functions via FFI + +There are 2 databases: + +- Frecency database (LMDB) that tracks file access patterns for scoring +- Query history database used to track the user's previous search queries + +### Key Files + +- `lua/fff.lua` - Entry point, delegates to main.lua +- `lua/fff/main.lua` - Public API (find_files, search, change_directory) +- `lua/fff/core.lua` - Initialization, autocmds, global state management +- `lua/fff/picker_ui.lua` - UI rendering, layout calculation, keymaps +- `lua/fff/file_picker/preview.lua` - File preview with syntax highlighting +- `lua/fff/file_picker/image.lua` - Image preview (snacks.nvim integration) +- `lua/fff/conf.lua` - Default config +- `lua/fff/rust/init.lua` - Loads compiled Rust shared library + +**Rust Side:** + +- `lua/fff/rust/lib.rs` - FFI bindings, global state (FILE_PICKER, FRECENCY) +- `lua/fff/rust/file_picker.rs` - Core FilePicker struct, indexing, background watcher +- `lua/fff/rust/frecency.rs` - Frecency database (LMDB) and scoring +- `lua/fff/rust/query_tracker.rs` - Search query history tracking +- `lua/fff/rust/score.rs` - Fuzzy match scoring with frizbee integration +- `lua/fff/rust/git.rs` - Git status caching and repository detection +- `lua/fff/rust/background_watcher.rs` - File system watcher thread + +### Scoring Algorithm + +Located at the score.rs file + +### Build System + +- `Cargo.toml` - Rust dependencies and build configuration (package name: `fff_nvim`) +- `rust-toolchain.toml` - Specifies Rust nightly toolchain with required components +- `Cross.toml` - Cross-compilation settings using Zig for Linux targets +- **CI/CD Workflows**: + - `.github/workflows/rust.yml` - Rust testing, formatting, and clippy checks + - `.github/workflows/release.yaml` - Automated multi-platform builds + - `.github/workflows/stylua.yaml` - Lua code formatting validation + - `.github/workflows/nix.yml` - Nix build validation +- **Cross-compilation Support**: Uses `cross` tool with Zig backend for efficient cross-compilation + +## Development Notes + +### Working with Rust Code + +- Prefer struct methods over functions +- If there is more than 2 impls in the file - create new file +- Smaller concise comments over giant comment blocks +- Do not add doc comments to the private functions/structs +- Be very careful around locking and better double check with the human if something is going to require potentially long lock on a mutex/rwlock + +### Working with lua code + +- Document the types of public functions in every module +- Use `vim.validate()` for validating user inputs in public functions +- Try to reuse as much of existing functions as possible +- When working on new features for the UI **IT IS EXTREMELY IMPORTANT** to keep the core functionality of navigating between files, selecting, and seeing the preview working as is. NEVER break anything from the core UI functionality, only add new features on top of the current UI. +- When making a large chunk of code make lua test that opens neovim at `~/dev/lightsource` and opens the picker to test the ui functionality across the actual code. +- When adding a new highlights or any new shortcuts and configurable UI options add them to the neovim config. AND IMPORTANT: update the README.md with the new configuration options. + +### UI rendering + +When working on the UI changeds IT IS EXTREMELY important for you to test it for both prompt_position="bottom" and prompt_position="top" as the rendering logic is different for both of them in both rust and lua world. When the prompt is positioed in the bottom everything should work the same way as the top but would be reversed in order. (though navigation is same for both) + +## Top level API that can not introduce breaking changes under any circumstance + +Top level rust, lua, C, and bun APIs can not be changed under any circumstance diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..6ef491a --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys 0.61.2", +] + +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.0", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fff-c" +version = "0.9.6" +dependencies = [ + "fff-query-parser", + "fff-search", + "git2", + "serde_json", +] + +[[package]] +name = "fff-grep" +version = "0.9.6" +dependencies = [ + "bstr", + "memchr", +] + +[[package]] +name = "fff-mcp" +version = "0.9.6" +dependencies = [ + "clap", + "fff-query-parser", + "fff-search", + "git2", + "mimalloc", + "rmcp", + "schemars", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "fff-notify-debouncer-full" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a4ebea7b8a2840cd59358bbf396f6f04313ce8eae84ac79703ce80298b8731" +dependencies = [ + "file-id", + "log", + "notify", + "notify-types", + "rustc-hash", + "walkdir", +] + +[[package]] +name = "fff-nvim" +version = "0.9.6" +dependencies = [ + "ahash", + "chrono", + "criterion", + "ctrlc", + "fff-query-parser", + "fff-search", + "git2", + "ignore", + "mimalloc", + "mlua", + "once_cell", + "rand 0.8.5", + "tracing", + "tracing-subscriber", + "zlob", +] + +[[package]] +name = "fff-python" +version = "0.9.6" +dependencies = [ + "fff-query-parser", + "fff-search", + "git2", + "pyo3", +] + +[[package]] +name = "fff-query-parser" +version = "0.9.6" +dependencies = [ + "criterion", + "zlob", +] + +[[package]] +name = "fff-search" +version = "0.9.6" +dependencies = [ + "ahash", + "aho-corasick", + "blake3", + "criterion", + "ctor", + "dirs", + "dunce", + "fff-grep", + "fff-notify-debouncer-full", + "fff-query-parser", + "git2", + "glidesort", + "globset", + "heed", + "ignore", + "libc", + "libmimalloc-sys", + "memchr", + "memmap2", + "mimalloc", + "neo_frizbee", + "notify", + "parking_lot", + "pathdiff", + "proptest", + "rand 0.8.5", + "rayon", + "regex", + "regex-syntax", + "serde", + "signal-hook-registry", + "smallvec", + "tempfile", + "thiserror 2.0.18", + "tracing", + "tracing-appender", + "tracing-subscriber", + "zlob", +] + +[[package]] +name = "file-id" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc6a637b6dc58414714eddd9170ff187ecb0933d4c7024d1abbd23a3cc26e9" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags 2.11.0", + "libc", + "libgit2-sys", + "log", + "url", +] + +[[package]] +name = "glidesort" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2e102e6eb644d3e0b186fc161e4460417880a0a0b87d235f2e5b8fb30f2e9e0" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "heed" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a56c94661ddfb51aa9cdfbf102cfcc340aa69267f95ebccc4af08d7c530d393" +dependencies = [ + "bitflags 2.11.0", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" +dependencies = [ + "bincode", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ignore" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inotify" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199" +dependencies = [ + "bitflags 2.11.0", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "cty", + "libc", +] + +[[package]] +name = "libredox" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +dependencies = [ + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lmdb-master-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864808e0b19fb6dd3b70ba94ee671b82fce17554cf80aeb0a155c65bb08027df" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mlua" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd36acfa49ce6ee56d1307a061dd302c564eee757e6e4cd67eb4f7204846fab" +dependencies = [ + "bstr", + "either", + "libc", + "mlua-sys", + "mlua_derive", + "num-traits", + "parking_lot", + "rustc-hash", + "rustversion", +] + +[[package]] +name = "mlua-sys" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1c3a7fc7580227ece249fd90aa2fa3b39eb2b49d3aec5e103b3e85f2c3dfc8" +dependencies = [ + "cc", + "cfg-if", + "libc", + "pkg-config", +] + +[[package]] +name = "mlua_derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465bddde514c4eb3b50b543250e97c1d4b284fa3ef7dc0ba2992c77545dbceb2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "neo_frizbee" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2f6120a8da26bea3587731072111062c5d8c51ca3a3a75a716bd8b735d5882" + +[[package]] +name = "nix" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "notify" +version = "9.0.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b44b771d4dd781ef14c84078693e67495da6b47f609f72e8a4da8420a861240e" +dependencies = [ + "bitflags 2.11.0", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "objc2-core-foundation", + "objc2-core-services", + "walkdir", + "windows-sys 0.61.2", + "xxhash-rust", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "objc2-core-services" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pastey" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bitflags 2.11.0", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "pyo3" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rmcp" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e" +dependencies = [ + "async-trait", + "base64", + "chrono", + "futures", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rmcp-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "chrono", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +dependencies = [ + "crossbeam-channel", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlob" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41cb327ac1b7e7e0d4514658500cb5734cd655edbe4e5ffeda69955da9028ee" +dependencies = [ + "bindgen", + "bitflags 2.11.0", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e333fd1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,73 @@ +[workspace] +members = [ + "crates/fff-c", + "crates/fff-core", + "crates/fff-mcp", + "crates/fff-nvim", + "crates/fff-python", + "crates/fff-query-parser", + "crates/fff-grep", +] + +resolver = "2" + +[workspace.dependencies] +fff-grep = { version = "0.9.6", path = "crates/fff-grep" } +fff-query-parser = { version = "0.9.6", path = "crates/fff-query-parser", default-features = false } + +# Shared dependencies +ahash = "0.8" +blake3 = "1.8.2" +chrono = { version = "0.4", features = ["serde"] } +ctrlc = "3.4.2" +dirs = "5.0" +dunce = "1.0" +# git2 - base config without TLS (each crate adds platform-specific TLS) +git2 = { version = "0.20.2", default-features = false, features = [ + "vendored-libgit2", +] } +glidesort = "0.1" +globset = "0.4" +heed = "0.22.0" +ignore = "0.4.22" +memmap2 = "0.9" +mimalloc = "0.1.47" +signal-hook-registry = "1.4" +zlob = { version = "=1.6.1" } + +mlua = { version = "0.11.1", features = ["module", "luajit"] } +neo_frizbee = { version = "0.11.0", features = ["match_end_col"] } +notify = { version = "9.0.0-rc.3" } +notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.4" } +once_cell = "1.20.2" +parking_lot = "0.12" +pathdiff = "0.2.1" +rayon = "1.8.0" +regex = "1.11" +regex-syntax = "0.8" +smallvec = { version = "1.13", features = ["const_generics", "union"] } +thiserror = "2.0.10" +tracing = "0.1" + +[profile.release] +opt-level = 3 +lto = "fat" +codegen-units = 1 +strip = "debuginfo" + +[profile.ci] +inherits = "release" +# we use lto=fat locally for better SIMD for the march=native but +# on CI when we cross compiling we should not exclude any cpu flags checking +lto = "thin" + +[profile.bench] +inherits = "release" + +# For Instruments / xctrace: release-level optimization but keep debuginfo +# and symbols so sampled frames resolve to real Rust names. +[profile.prof] +inherits = "release" +debug = "full" +strip = false +lto = "thin" diff --git a/Formula/fff-mcp.rb b/Formula/fff-mcp.rb new file mode 100644 index 0000000..80a357b --- /dev/null +++ b/Formula/fff-mcp.rb @@ -0,0 +1,60 @@ +# Originally authored by @jellydn (https://github.com/jellydn/homebrew-tap). +# Maintained in-repo; auto-bumped by .github/workflows/release.yaml on stable releases. +class FffMcp < Formula + desc "Fast file search toolkit for AI agents (MCP server)" + homepage "https://github.com/dmtrKovalenko/fff.nvim" + license "MIT" + version "0.9.6" + + LIVECHECK_REPO = "dmtrKovalenko/fff.nvim".freeze + RELEASE_BASE = "https://github.com/dmtrKovalenko/fff.nvim/releases/download".freeze + + on_macos do + on_arm do + url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-apple-darwin" + sha256 "29a7fadeafb062f3e5954b1ab8c69e14dca24f5e061cd8d3b1ea1bab385a3754" + end + + on_intel do + url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-apple-darwin" + sha256 "58259324c2c13a1b6f24f13138c2cd3eae9ff20e05201a539beb8f2044a651aa" + end + end + + on_linux do + on_arm do + url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-unknown-linux-gnu" + sha256 "91e6fa14e040588dc92de854e35020536f1e2458ce3386b2b727b2e7a88f6684" + end + + on_intel do + url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-unknown-linux-gnu" + sha256 "d1bd2b89a79e8eda71b1754260499cec1feaafd2adf372e92371c8d6b68509a3" + end + end + + livecheck do + url "https://github.com/#{LIVECHECK_REPO}/releases/latest" + strategy :github_latest + end + + def install + if OS.mac? + if Hardware::CPU.arm? + bin.install "fff-mcp-aarch64-apple-darwin" => "fff-mcp" + elsif Hardware::CPU.intel? + bin.install "fff-mcp-x86_64-apple-darwin" => "fff-mcp" + end + elsif OS.linux? + if Hardware::CPU.arm? + bin.install "fff-mcp-aarch64-unknown-linux-gnu" => "fff-mcp" + elsif Hardware::CPU.intel? + bin.install "fff-mcp-x86_64-unknown-linux-gnu" => "fff-mcp" + end + end + end + + test do + system bin/"fff-mcp", "--healthcheck" + end +end diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ea495f0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Dmitriy Kovalenko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a3004f4 --- /dev/null +++ b/Makefile @@ -0,0 +1,348 @@ +PLENARY_DIR ?= ../plenary.nvim +MINI_DIR ?= ../mini.nvim + +PREFIX ?= /usr/local +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR ?= $(PREFIX)/include + +# Compile-time cfg that gates the watcher + git-status fuzz stress test. +STRESS_RUSTFLAGS := --cfg stress +FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE + +SHELL := bash +# Order matters: `-c` must be last so bash treats the recipe as the script +# string rather than the literal `-o` / `pipefail` tokens. +.SHELLFLAGS := -o pipefail -euc + +.PHONY: build build-c-lib install uninstall test test-rust test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile + +all: format test lint + +SYNC_API_SRC := packages/shared/fff-api.ts +SYNC_API_TARGETS := packages/fff-node/src/fff-api.ts packages/fff-bun/src/fff-api.ts +SYNC_API_BANNER := // ----------------------------------------------------------------------------\n// GENERATED FILE - DO NOT EDIT.\n// Copied from: ${SYNC_API_SRC}\n// Run make sync-js-api from the repo root to regenerate.\n// ----------------------------------------------------------------------------\n\n + +sync-js-api: + @for target in $(SYNC_API_TARGETS); do \ + printf '$(SYNC_API_BANNER)' > "$$target"; \ + cat $(SYNC_API_SRC) >> "$$target"; \ + echo "synced: $$target"; \ + done + +sync-js-api-check: + @status=0; \ + for target in $(SYNC_API_TARGETS); do \ + tmp=$$(mktemp); \ + printf '$(SYNC_API_BANNER)' > "$$tmp"; \ + cat $(SYNC_API_SRC) >> "$$tmp"; \ + if ! cmp -s "$$tmp" "$$target"; then \ + echo "out of date: $$target (run make sync-js-api)"; status=1; \ + fi; \ + rm -f "$$tmp"; \ + done; \ + exit $$status + +build: + cargo build --release --no-default-features --features zlob + +build-c-lib: + cargo build --release -p fff-c --no-default-features --features zlob + +header: + cbindgen --config crates/fff-c/cbindgen.toml --crate fff-c --output crates/fff-c/include/fff.h + +# Install the C library and header under $(PREFIX) (default /usr/local). +# Override PREFIX for user-local installs, e.g. `make install PREFIX=$$HOME/.local`. +# DESTDIR is honoured for packagers. +install: build-c-lib + install -d $(DESTDIR)$(LIBDIR) + install -d $(DESTDIR)$(INCLUDEDIR) + install -m 0644 crates/fff-c/include/fff.h $(DESTDIR)$(INCLUDEDIR)/fff.h + @if [ -f target/release/libfff_c.dylib ]; then \ + install -m 0755 target/release/libfff_c.dylib $(DESTDIR)$(LIBDIR)/libfff_c.dylib; \ + echo "Installed $(DESTDIR)$(LIBDIR)/libfff_c.dylib"; \ + fi + @if [ -f target/release/libfff_c.so ]; then \ + install -m 0755 target/release/libfff_c.so $(DESTDIR)$(LIBDIR)/libfff_c.so; \ + echo "Installed $(DESTDIR)$(LIBDIR)/libfff_c.so"; \ + fi + @if [ -f target/release/fff_c.dll ]; then \ + install -m 0755 target/release/fff_c.dll $(DESTDIR)$(LIBDIR)/fff_c.dll; \ + echo "Installed $(DESTDIR)$(LIBDIR)/fff_c.dll"; \ + fi + @echo "Installed header $(DESTDIR)$(INCLUDEDIR)/fff.h" + +uninstall: + rm -f $(DESTDIR)$(LIBDIR)/libfff_c.dylib + rm -f $(DESTDIR)$(LIBDIR)/libfff_c.so + rm -f $(DESTDIR)$(LIBDIR)/fff_c.dll + rm -f $(DESTDIR)$(INCLUDEDIR)/fff.h + @echo "Removed fff-c from $(DESTDIR)$(PREFIX)" + +test-setup: + @if [ ! -d "$(PLENARY_DIR)" ]; then \ + echo "Cloning plenary.nvim..."; \ + git clone --depth 1 https://github.com/nvim-lua/plenary.nvim $(PLENARY_DIR); \ + fi + @if [ ! -d "$(MINI_DIR)" ]; then \ + echo "Cloning mini.nvim..."; \ + git clone --depth 1 https://github.com/echasnovski/mini.nvim $(MINI_DIR); \ + fi + +test-rust: + cargo test --workspace --no-default-features --features zlob --exclude fff-nvim + +CC ?= cc +CFLAGS ?= -O0 -g -Wall -Wextra -std=c99 +TARGET_DIR ?= target/release +SMOKE_BIN := $(TARGET_DIR)/fff_c_smoke +SMOKE_SRC := crates/fff-c/tests/smoke.c +SMOKE_INCLUDE := crates/fff-c/include + +test-c-smoke: build-c-lib + $(CC) $(CFLAGS) -I $(SMOKE_INCLUDE) -L $(TARGET_DIR) \ + -Wl,-rpath,@loader_path/../target/release \ + -Wl,-rpath,$$(pwd)/$(TARGET_DIR) \ + $(SMOKE_SRC) -lfff_c -o $(SMOKE_BIN) + $(SMOKE_BIN) . + +# Alias kept for the `external-tests.yml` workflow naming. +test-c-api: test-c-smoke + +# neovim instance swallows internal crashes and doesn't rise the the error exiting silently +# so check the stdout in case the sigsegv coming out of fff was printed (actual regression). +# Output is streamed live via `tee`; pipefail (set above) propagates nvim's exit. +test-lua: test-setup build + @logfile=$$(mktemp); \ + trap 'rm -f "$$logfile"' EXIT; \ + nvim --headless -u tests/minimal_init.lua \ + -c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1 \ + | tee "$$logfile"; \ + if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \ + echo ""; \ + echo "FAIL: native crash detected during lua tests"; \ + exit 1; \ + fi + +test-lua-snap: test-setup build + @logfile=$$(mktemp); \ + trap 'rm -f "$$logfile"' EXIT; \ + nvim --headless -u tests/minimal_init.lua \ + -c "lua local ok,err=pcall(require('mini.test').run_file,'tests/picker_ui_snap.lua'); if not ok then io.stderr:write('mini.test failed to load: '..tostring(err)..'\\n'); vim.cmd('cquit 2') end" 2>&1 \ + | tee "$$logfile"; \ + if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \ + echo ""; \ + echo "FAIL: native crash detected during snapshot tests"; \ + exit 1; \ + fi + +test-version: test-setup + nvim --headless -u tests/minimal_init.lua \ + -c "PlenaryBustedFile tests/version_spec.lua" 2>&1 + +prepare-bun: build sync-js-api + mkdir -p packages/fff-bun/bin + cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null || true; \ + cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null || true; \ + cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null || true + +prepare-node: build sync-js-api + mkdir -p packages/fff-node/bin + cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null || true; \ + cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null || true; \ + cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null || true + +test-bun: prepare-bun + cd packages/fff-bun && bun test test/ + cd packages/pi-fff && bun test test/ + +# Same as prepare-bun but puts the compiled binary into the actual npm package location +prepare-bun-packaged: prepare-bun + @machine=$$(uname -m); \ + case "$$machine" in \ + x86_64|amd64) arch=x64 ;; \ + aarch64|arm64) arch=arm64 ;; \ + *) echo "unsupported arch: $$machine" >&2; exit 1 ;; \ + esac; \ + case "$$(uname -s)" in \ + Darwin) lib=libfff_c.dylib; pkg=fff-bin-darwin-$$arch ;; \ + Linux) lib=libfff_c.so; \ + if ldd --version 2>&1 | grep -qi musl; then libc=musl; else libc=gnu; fi; \ + pkg=fff-bin-linux-$$arch-$$libc ;; \ + MINGW*|MSYS*|CYGWIN*|Windows_NT) lib=fff_c.dll; pkg=fff-bin-win32-$$arch ;; \ + *) echo "unsupported OS: $$(uname -s)" >&2; exit 1 ;; \ + esac; \ + src=target/release/$$lib; \ + [ -f "$$src" ] || { echo "missing built library: $$src" >&2; exit 1; }; \ + dest=packages/fff-bun/node_modules/@ff-labs/$$pkg; \ + rm -rf "$$dest"; mkdir -p "$$dest"; \ + cp "$$src" "$$dest/$$lib"; \ + printf '{ "name": "@ff-labs/%s", "version": "0.0.0", "main": "%s" }\n' "$$pkg" "$$lib" > "$$dest/package.json" + +# Compile a bun example to a standalone executable and run it. Verifies the +# native libfff_c is embedded + loaded from a `bun build --compile` binary. +# The staged bin package is removed before running so success proves the lib +# was embedded, not resolved from disk. +test-bun-compile: prepare-bun-packaged + cd packages/fff-bun && \ + if [ "$$(uname -s)" = "Linux" ]; then \ + if ldd --version 2>&1 | grep -qi musl; then DEFINE='--define FFF_LIBC="musl"'; \ + else DEFINE='--define FFF_LIBC="gnu"'; fi; \ + else DEFINE=""; fi; \ + bun build --compile $$DEFINE ./examples/glob-bench.ts --outfile ./glob-bench-bin && \ + EXE=./glob-bench-bin; [ -f "$$EXE.exe" ] && EXE="$$EXE.exe"; \ + rm -rf bin node_modules/@ff-labs; \ + "$$EXE" . '**/*.ts' 1 | tee /tmp/fff-compile-e2e.log && \ + grep -q 'fff.glob' /tmp/fff-compile-e2e.log + rm -f packages/fff-bun/glob-bench-bin packages/fff-bun/glob-bench-bin.exe + +test-node: prepare-node + cd packages/fff-node && npm run build && node test/e2e.mjs + +test-js: test-bun test-node + +# Bug pinning stress test script over fff-node for issue #515 +# Just keep it untouched because it's good enough + some stress for SDK +FFF_STRESS_ITERS ?= 50 +test-node-stress: prepare-node + cd packages/fff-node && npm run build && \ + FFF_STRESS_ITERS=$(FFF_STRESS_ITERS) node test/stress-515.mjs + +test: test-rust test-lua test-lua-snap test-version test-bun test-node test-node-stress + +test-stress-seeded: + FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \ + RUSTFLAGS="$(STRESS_RUSTFLAGS)" \ + cargo test --release \ + -p fff-search \ + --test fuzz_git_watcher_stress \ + --no-default-features --features zlob \ + -- --nocapture stress_seeded + +test-stress-random: + RUSTFLAGS="$(STRESS_RUSTFLAGS)" \ + cargo test --release \ + -p fff-search \ + --test fuzz_git_watcher_stress \ + --no-default-features --features zlob \ + -- --nocapture stress_random + +test-stress-regressions: + RUSTFLAGS="$(STRESS_RUSTFLAGS)" \ + cargo test --release \ + -p fff-search \ + --test fuzz_git_watcher_stress \ + --no-default-features --features zlob \ + -- --nocapture stress_regression stress_merge_conflict_convergence + +test-stress-repos: + RUSTFLAGS="$(STRESS_RUSTFLAGS)" \ + cargo test --release \ + -p fff-search \ + --test fuzz_real_repos \ + --no-default-features --features zlob \ + -- --nocapture + +test-stress: test-stress-seeded test-stress-random test-stress-regressions test-stress-repos + +# Update version in a package.json, including optionalDependencies. +# Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234 +set-npm-version: + @test -n "$(PKG)" || (echo "PKG is required" && exit 1) + @test -n "$(VERSION)" || (echo "VERSION is required" && exit 1) + node -e " \ + const fs = require('fs'); \ + const pkg = JSON.parse(fs.readFileSync('$(PKG)/package.json', 'utf8')); \ + pkg.version = '$(VERSION)'; \ + if (pkg.optionalDependencies) { \ + for (const dep of Object.keys(pkg.optionalDependencies)) { \ + pkg.optionalDependencies[dep] = '$(VERSION)'; \ + } \ + } \ + fs.writeFileSync('$(PKG)/package.json', JSON.stringify(pkg, null, 2) + '\n'); \ + " + @echo "Set $(PKG) to $(VERSION)" + +format-rust: + cargo fmt --all +format-lua: + stylua . +format-ts: + bun format + +format: format-rust format-lua format-ts + +lint-rust: + cargo clippy --workspace --no-default-features --features zlob -- -D warnings +lint-lua: + ~/.luarocks/bin/luacheck . +lint-ts: + bun lint + +lint: lint-rust lint-lua lint-ts + +check: format lint + +FFF_RELEASE_REPO ?= dmtrKovalenko/fff.nvim +FFF_FORMULA_PATH ?= Formula/fff-mcp.rb +FFF_INSTALL_SCRIPT_PATH ?= install-mcp.sh + +# Read the sha256 for $1 (filename, no .sha256 suffix). Reads from +# BINARIES_DIR/$1.sha256 when set; otherwise curls the GitHub release. +define fff_fetch_sha + if [ -n "$$BINARIES_DIR" ]; then \ + awk '{print $$1}' "$$BINARIES_DIR/$$1.sha256" \ + || { echo "Missing checksum file: $$BINARIES_DIR/$$1.sha256" >&2; exit 1; }; \ + else \ + curl -fsSL "https://github.com/$(FFF_RELEASE_REPO)/releases/download/v$(VERSION)/$$1.sha256" \ + | awk '{print $$1}'; \ + fi +endef + +bump-homebrew-formula: + @test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-homebrew-formula VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1) + @export BINARIES_DIR="$(BINARIES_DIR)"; \ + fetch_sha() { $(fff_fetch_sha); }; \ + sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \ + sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \ + sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-gnu)"; \ + sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-gnu)"; \ + sed -i.bak \ + -e 's/^ version "[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"$$/ version "$(VERSION)"/' \ + -e '/fff-mcp-aarch64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_arm"'"/;}' \ + -e '/fff-mcp-x86_64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_intel"'"/;}' \ + -e '/fff-mcp-aarch64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_arm"'"/;}' \ + -e '/fff-mcp-x86_64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_intel"'"/;}' \ + "$(FFF_FORMULA_PATH)" && rm -f "$(FFF_FORMULA_PATH).bak"; \ + echo "Bumped $(FFF_FORMULA_PATH) to v$(VERSION)" + +bump-install-mcp-sh: + @test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-install-mcp-sh VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1) + @export BINARIES_DIR="$(BINARIES_DIR)"; \ + fetch_sha() { $(fff_fetch_sha); }; \ + sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-musl)"; \ + sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-musl)"; \ + sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \ + sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \ + sha_win_intel="$$(fetch_sha fff-mcp-x86_64-pc-windows-msvc.exe)"; \ + sha_win_arm="$$(fetch_sha fff-mcp-aarch64-pc-windows-msvc.exe)"; \ + sed -i.bak \ + -e 's|^PINNED_RELEASE_TAG=".*"|PINNED_RELEASE_TAG="v$(VERSION)"|' \ + -e 's|^SHA256_X86_64_UNKNOWN_LINUX_MUSL=".*"|SHA256_X86_64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_intel"'"|' \ + -e 's|^SHA256_AARCH64_UNKNOWN_LINUX_MUSL=".*"|SHA256_AARCH64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_arm"'"|' \ + -e 's|^SHA256_X86_64_APPLE_DARWIN=".*"|SHA256_X86_64_APPLE_DARWIN="'"$$sha_darwin_intel"'"|' \ + -e 's|^SHA256_AARCH64_APPLE_DARWIN=".*"|SHA256_AARCH64_APPLE_DARWIN="'"$$sha_darwin_arm"'"|' \ + -e 's|^SHA256_X86_64_PC_WINDOWS_MSVC=".*"|SHA256_X86_64_PC_WINDOWS_MSVC="'"$$sha_win_intel"'"|' \ + -e 's|^SHA256_AARCH64_PC_WINDOWS_MSVC=".*"|SHA256_AARCH64_PC_WINDOWS_MSVC="'"$$sha_win_arm"'"|' \ + "$(FFF_INSTALL_SCRIPT_PATH)" && rm -f "$(FFF_INSTALL_SCRIPT_PATH).bak"; \ + echo "Bumped $(FFF_INSTALL_SCRIPT_PATH) tag + checksums to v$(VERSION)" + +CRATES_TO_PUBLISH= fff-grep fff-query-parser fff-search + +publish-crates: + @test -n "$(V)" || (echo "V is required. Usage: make publish-crates V=0.2.0" && exit 1) + cargo install cargo-edit --force --locked + cargo set-version $(V) || exit 1; + @for crate in $(CRATES_TO_PUBLISH); do \ + cargo publish -p $$crate --allow-dirty $$(if [ -n "$$CI" ]; then echo "--no-verify"; fi) || exit 1; \ + done diff --git a/README.md b/README.md new file mode 100644 index 0000000..c7bb11e --- /dev/null +++ b/README.md @@ -0,0 +1,825 @@ +FFF + +

+ A file search toolkit for humans and AI agents. Really fast. +

+ +Typo-resistant path and content search, frecency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once. + +Powers file search in [opencode](http://github.com/anomalyco/opencode/), [nushell](https://github.com/nushell/nushell), and many more amazing projects! + +Originally started as [Neovim plugin](#neovim-plugin) people loved, but it turned out that plenty of AI harnesses and code editors need the same thing: accurate, fast file search as a library. That is what fff is. + +--- + +Pick what you are interested in: + +
+ +

MCP server

+ + +Works with Claude Code, Codex, OpenCode, Cursor, Cline, and any MCP-capable client. Fewer grep roundtrips, less wasted context, faster answers. + +![Benchmark chart comparing FFF against the built-in AI file-search tools](./chart.png) + +### One-line install + +Linux / macOS: + +```bash +curl -L https://dmtrkovalenko.dev/install-fff-mcp.sh | bash +``` + +Windows (PowerShell): + +```powershell +irm https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.ps1 | iex +``` + +The scripts live at [`install-mcp.sh`](./install-mcp.sh) and [`install-mcp.ps1`](./install-mcp.ps1) if you want to read them first. They print the exact wiring instructions for your client. + +### Homebrew (macOS / Linux) + +```bash +brew install dmtrKovalenko/fff/fff-mcp +brew upgrade fff-mcp # after new stable releases +``` + +Formula lives in [`Formula/fff-mcp.rb`](./Formula/fff-mcp.rb) in this repo and is **auto-bumped on every stable release** (see `bump-homebrew-formula` in [`.github/workflows/release.yaml`](./.github/workflows/release.yaml)). Installs the prebuilt `fff-mcp` binary from [GitHub releases](https://github.com/dmtrKovalenko/fff.nvim/releases). + +Once the server is connected, ask the agent to "use fff" and it picks up the `ffgrep`, `fffind`, and `fff-multi-grep` tools. + +### Recommended agent prompt + +Drop this into your project's `CLAUDE.md` or equivalent: + +```markdown +For any file search or grep in the current git-indexed directory, use fff tools. +``` + +### What changes + +- Frecency memory. Files you actually open rank higher next time. Warm-up from git touch history runs automatically. +- Definition-first hinting. Lines that look like code definitions are classified on the Rust side, no regex overhead in your prompt. +- Smart-case with auto-fuzzy fallback. `IsOffTheRecord` finds snake_case variants; zero-match queries retry as fuzzy and surface the best approximate hits. +- Git-aware annotations. Modified, untracked, and staged files are tagged so the agent reaches for what you are actively changing. + +Source: [`crates/fff-mcp/`](./crates/fff-mcp/). + +
+ +The MCP server gives any agent a file search tool that is faster and more token-efficient than the built-in one. + +
+ +

Pi agent extension

+ + +### Install + +```bash +pi install npm:@ff-labs/pi-fff +``` + +### Modes + +Three operating modes, switchable at runtime with `/fff-mode`: + +| Mode | What it does | +| ------------------------ | --------------------------------------------------------------------------------- | +| `tools-and-ui` (default) | Adds `ffgrep` and `fffind` tools, replaces `@`-mention autocomplete with FFF. | +| `tools-only` | Only tool injection. Keeps pi's native editor autocomplete. | +| `override` | Replaces pi's built-in `grep`, `find`, and `multi_grep` with FFF implementations. | + +Env vars: `PI_FFF_MODE`, `FFF_FRECENCY_DB`, `FFF_HISTORY_DB`. Flags: `--fff-mode`, `--fff-frecency-db`, `--fff-history-db`. + +### Agent-facing tools + +- `ffgrep`. Content search. Accepts `path`, `exclude` (comma, space, or array; leading `!` optional), `caseSensitive`, `context`, and cursor pagination. Auto-detects regex, falls back to fuzzy on zero exact matches, rejects `.*`-style wildcard-only patterns up front. +- `fffind`. Path and filename search. Matches the whole repo-relative path, not just the filename. Frecency-aware. The weak-match detector flags scattered fuzzy noise before it floods the agent's context. + +### Commands + +- `/fff-mode [tools-and-ui | tools-only | override]`. Show or switch the mode. +- `/fff-health`. Picker, frecency, and git integration status. +- `/fff-rescan`. Force a rescan. + +Source: [`packages/pi-fff/`](./packages/pi-fff/). + +
+ +The Pi extension swaps pi's native tools for FFF implementations and feeds the interactive editor's `@`-mention autocomplete from the frecency-ranked index. + +
+ +

fff.nvim

+ + +Demo on the Linux kernel repo (100k files, 8GB): + +https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb + +### Installation + +#### lazy.nvim + +```lua +{ + 'dmtrKovalenko/fff.nvim', + build = function() + -- downloads a prebuilt binary or falls back to cargo build + require("fff.download").download_or_build_binary() + end, + -- for nixos: + -- build = "nix run .#release", + opts = { + debug = { + enabled = true, + show_scores = true, + }, + }, + lazy = false, -- the plugin lazy-initialises itself + keys = { + { "ff", function() require('fff').find_files() end, desc = 'FFFind files' }, + { "fg", function() require('fff').live_grep() end, desc = 'LiFFFe grep' }, + { "fz", + function() require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) end, + desc = 'Live fffuzy grep', + }, + { "fw", + function() require('fff').live_grep_under_cursor() end, + mode = { 'n', 'x' }, + desc = 'Search current word / selection', + }, + }, +} +``` + +#### vim.pack + +```lua +vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' }) + +vim.api.nvim_create_autocmd('PackChanged', { + callback = function(ev) + local name, kind = ev.data.spec.name, ev.data.kind + if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then + if not ev.data.active then vim.cmd.packadd('fff.nvim') end + require('fff.download').download_or_build_binary() + end + end, +}) + +vim.g.fff = { + lazy_sync = true, + debug = { enabled = true, show_scores = true }, +} + +vim.keymap.set('n', 'ff', function() require('fff').find_files() end, { desc = 'FFFind files' }) +``` + +### Public API + +```lua +require('fff').find_files() -- find files in current repo +require('fff').live_grep() -- live content grep +require('fff').live_grep_under_cursor() -- grep in normal, selection in visual +require('fff').scan_files() -- force rescan +require('fff').refresh_git_status() -- refresh git status +require('fff').find_files_in_dir(path) -- find in a specific dir +require('fff').change_indexing_directory(new_path) -- change root + +-- Programmatic search (no UI). Useful for plugin integrations. +require('fff').file_search(query, opts) -- fuzzy search files / dirs / mixed +require('fff').content_search(query, opts) -- programmatic grep +``` + +#### `file_search(query, opts)` + +Returns a structured result `{ items, scores, total_matched, total_files?, total_dirs?, location? }`. Each item has a `type` field (`"file"` or `"directory"`) and `name` / `relative_path`. File items also expose `size`, `modified`, `git_status`, `is_binary`, and frecency scores. + +```lua +local r = require('fff').file_search('button', { + mode = 'mixed', -- 'files' (default) | 'directories' | 'mixed' + max_results = 50, + page = 0, -- 0-based pagination + current_file = nil, -- path to deprioritize for distance scoring + max_threads = 4, + cwd = nil, -- switch indexed root if different (see below) + wait_for_index_ms = nil, -- override the default scan wait timeout +}) +for _, item in ipairs(r.items) do + print(item.type, item.relative_path) +end +``` + +#### `content_search(query, opts)` + +Returns a `GrepResult` `{ items, total_matched, total_files_searched, total_files, filtered_file_count, next_file_offset, regex_fallback_error? }`. Each match item has `relative_path`, `name`, `line_number`, `col`, `line_content`, `match_ranges`, plus the same file metadata as `file_search`. + +```lua +local r = require('fff').content_search('TODO', { + mode = 'plain', -- 'plain' (default) | 'regex' | 'fuzzy' + max_file_size = 10 * 1024 * 1024, + max_matches_per_file = 100, + smart_case = true, + page_size = 50, + file_offset = 0, + time_budget_ms = 0, + trim_whitespace = false, + cwd = nil, -- switch indexed root if different + wait_for_index_ms = nil, -- override the default scan wait timeout +}) +for _, m in ipairs(r.items) do + print(string.format('%s:%d %s', m.relative_path, m.line_number, m.line_content)) +end +``` + +Both functions accept the same constraint syntax as the UI pickers (e.g. `git:modified`, `*.rs`, `!test/`, glob patterns). + +#### `cwd` and indexing + +Both `file_search` and `content_search` honour an optional `cwd` field. The first call to either function lazily initialises the picker at `config.base_path` (your Neovim cwd by default). + +- If `cwd` matches the currently indexed root, the call returns immediately against the existing index. +- If `cwd` differs, the picker is re-indexed at the new root and the call **blocks** (default up to 10 s) until the new picker is installed and its initial scan completes — so callers always get results from the right tree. +- If the index is still warming up after a `change_indexing_directory`, you can pass `wait_for_index_ms = N` to block for up to `N` ms regardless of whether `cwd` triggered the swap. Pass `0` to skip waiting entirely (useful for fire-and-forget calls where partial results are acceptable). +- Invalid or non-existent `cwd` paths return an empty result and emit an error via `vim.notify`. + +### Commands + +- `:FFFScan`. Rescan files. +- `:FFFRefreshGit`. Refresh git status. +- `:FFFClearCache [all|frecency|files]`. Clear caches. +- `:FFFHealth`. Health check. +- `:FFFDebug [on|off|toggle]`. Toggle the scoring display. +- `:FFFOpenLog`. Open `~/.local/state/nvim/log/fff.log`. + +### Configuration + +Defaults are sensible. Override only what you care about. + +```lua +require('fff').setup({ + base_path = vim.fn.getcwd(), + prompt = '> ', + title = 'FFFiles', + max_results = 100, + max_threads = 4, + lazy_sync = true, + prompt_vim_mode = false, + follow_symlinks = false, + -- Allow indexing the user's $HOME directory. Enabled by default. + -- Disable if you strictly sure you don't want this, as it makes whole fff error hard + enable_home_dir_scanning = true, + -- Allow indexing a filesystem root (e.g. `/`, `C:\`). Disabled by default + enable_fs_root_scanning = false, + layout = { + height = 0.8, + width = 0.8, + prompt_position = 'bottom', -- or 'top' + preview_position = 'right', -- 'left' | 'right' | 'top' | 'bottom' + preview_size = 0.5, + -- Border style for the picker windows. Leave unset (nil) to follow the + -- global `vim.o.winborder`; set it to override fff's borders independently. + border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none' + flex = { size = 130, wrap = 'top' }, + min_list_height = 10, -- do not display anything except the list below this threshold + show_scrollbar = true, + path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end' | 'start' + anchor = 'center', + }, + preview = { + enabled = true, + max_size = 10 * 1024 * 1024, + chunk_size = 8192, + binary_file_threshold = 1024, + imagemagick_info_format_str = '%m: %wx%h, %[colorspace], %q-bit', + line_numbers = false, + cursorlineopt = 'both', + wrap_lines = false, + filetypes = { + svg = { wrap_lines = true }, + markdown = { wrap_lines = true }, + text = { wrap_lines = true }, + }, + }, + keymaps = { + close = '', + select = '', + select_split = '', + select_vsplit = '', + select_tab = '', + move_up = { '', '' }, + move_down = { '', '' }, + preview_scroll_up = '', + preview_scroll_down = '', + toggle_debug = '', + cycle_grep_modes = '', + -- grep mode only: jump cursor to first match of next/prev file group + grep_jump_to_next_file = { '', '' }, + grep_jump_to_prev_file = { '', '' }, + cycle_previous_query = '', + toggle_select = '', + send_to_quickfix = '', + focus_list = 'l', + focus_preview = 'p', + }, + frecency = { + enabled = true, + db_path = vim.fn.stdpath('cache') .. '/fff_nvim', + }, + history = { + enabled = true, + db_path = vim.fn.stdpath('data') .. '/fff_queries', + min_combo_count = 3, + combo_boost_score_multiplier = 100, + }, + git = { + status_text_color = false, -- true to color filenames by git status + }, + select = { + -- Return winid to open the chosen file in, or nil to open in the original window + select_window = function(current_buf, action) --[[ default impl ]] end, + }, + grep = { + max_file_size = 10 * 1024 * 1024, + max_matches_per_file = 100, + smart_case = true, + time_budget_ms = 150, + modes = { 'plain', 'regex', 'fuzzy' }, + trim_whitespace = false, + enable_filename_constraint = false, -- treat filename-like tokens (e.g. `score.rs`) in a grep query as a file-path filter scoping the search; off = searched as literal text + location_format = ':%d:%d', -- printf format for line:col prefix in grep results, e.g. ':%d' for line-only + }, + debug = { + enabled = false, -- show the file info panel next to the preview + show_scores = false, -- inline scores in the file list + -- Per-section toggles for the file info panel. Accepts a boolean shorthand + -- (`show_file_info = true|false`) to flip everything at once. The panel + -- adapts to width: narrow renders sections vertically, wide renders them + -- as a two-column grid. Disable a section to also shrink the panel. + show_file_info = { + file_info = true, -- size, type, git status, frecency + score_breakdown = true, -- total + match type, bonuses, modifiers, penalty + -- modified + accessed timestamps; pass a table to hide individual rows: + -- timings = { modified = false, accessed = true } + timings = true, + full_path = true, -- relative path at the bottom (wraps if too long) + }, + }, + logging = { + -- logs will be written in a parent directory of this file path in files like + -- `++.`. Run :FFFOpenLog to open current one + log_file = vim.fn.stdpath('log') .. '/fff.log', + log_level = 'info', + retain_runs = 20, + }, +}) +``` + +### Live grep modes + +`` cycles between `plain`, `regex`, and `fuzzy`. The list is configurable via `grep.modes`, and single-mode setups hide the indicator entirely. + +Per-call override: + +```lua +require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) +require('fff').live_grep({ query = 'search term' }) -- pre-fill +``` + +### Constraints + +Both find and grep accept these tokens to refine a query: + +- `git:modified`. One of `modified`, `staged`, `deleted`, `renamed`, `untracked`, `ignored`. +- `test/`. Any deeply nested children of `test/`. +- `!something`, `!test/`, `!git:modified`. Exclusion. +- `./**/*.{rs,lua}`. Any valid glob, powered by [zlob](https://github.com/dmtrKovalenko/zlob). + +Grep-only: + +- `*.md`, `*.{c,h}`. Extension filter. +- `src/main.rs`. Grep inside a single file. + +Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`. + +### Open in invoking window + +By default fff.nvim will try to open a file in the most suitable window, so any non-file buffers are not affected. You can customize or disable this by providing: + +```lua +require('fff').setup({ + select = { + select_window = function(_current_buf, _action) return nil end, + }, +}) +``` + +Caveat: the chosen file replaces the buffer in the invoking window even if it's a non-modifiable / special buftype. `winfixbuf` windows still fall back to `:split` to avoid `E1513`. + +### Multi-select and quickfix + +- ``. Toggle selection (shows a thick `▊` in the signcolumn). +- ``. Send selected files to the quickfix list and close the picker. + +### Git status highlighting + +Sign-column indicators are on by default. To color filename text by git status, set `git.status_text_color = true` and adjust the `hl.git_*` groups. See `:help fff.nvim` for the full list. + +### Float colors + +The picker maps its float content to `NormalFloat` (via `hl.normal`) and the border to `FloatBorder`. Default `FloatBorder` links to `NormalFloat`, so border and content share a background out of the box and the picker reads as a single popup. Override `hl.normal = 'Normal'` to make the picker blend with the editor instead. + +For finer control, set `hl.winhl` to override the per-window `winhighlight`. It accepts either a single string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, and `file_info` keys. Missing keys fall back to the default built from `hl.normal`, `hl.border`, and `hl.title`. + +```lua +-- Apply the same winhighlight to all picker windows +hl = { winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title' } + +-- Or override specific windows only +hl = { + winhl = { + prompt = 'Normal:Pmenu,FloatBorder:FloatBorder', + list = 'Normal:NormalFloat,FloatBorder:FloatBorder', + preview = 'Normal:NormalFloat,FloatBorder:FloatBorder', + }, +} +``` + +### File info panel + +Enable with `debug.enabled = true`. The panel sits above the preview and shows +file metadata, score breakdown, timestamps and the full absolute path. It +adapts to the panel width: at narrow widths sections stack vertically (B2), +at wide widths sections render as a two-column grid (H2). Each section can be +disabled individually via `debug.show_file_info`. + +Customise the panel via `hl`: + +| key | default | used for | +| ----------------------- | ----------------- | ---------------------------------- | +| `file_info_section` | `Title` | section header label | +| `file_info_separator` | `FloatBorder` | dashes that act as section borders | +| `file_info_label` | `Comment` | row labels (Size, Type, Git, ...) | +| `file_info_value` | `Normal` fg | plain values | +| `file_info_value_dim` | `NonText` | dim values, separators inside rows | +| `file_info_size` | `Number` | file size value | +| `file_info_type` | `Type` | filetype value | +| `file_info_path` | `Directory` | full path | +| `file_info_total_score` | bold + `Number` | total score (bold) | +| `file_info_match_type` | bold + `Special` | match type (bold) | +| `file_info_score_pos` | `DiagnosticOk` | positive score components | +| `file_info_score_neg` | `DiagnosticError` | negative score components | + +### File filtering + +FFF honours `.gitignore`. For picker-only ignores that do not touch git, add a sibling `.ignore` file: + +```gitignore +*.md +docs/archive/**/*.md +``` + +Run `:FFFScan` to force a rescan. + +### Troubleshooting + +- `:FFFHealth` verifies picker init, optional dependencies, and DB connectivity. +- `:FFFOpenLog` opens the current session's log file. +- Historical log files are stored near the main log file `/log/fff++.log` (up to 20 files) +- For a crash backtrace, run `lldb -- nvim` or `gdb -- nvim` and reproduce + +
+ +The best file search picker for neovim. Period. Faster and more intuitive queries, frecency ranking, definition classification and much more. + +
+ +

Node & Bun SDK

+ + +```bash +npm install @ff-labs/fff-node +# or +bun add @ff-labs/fff-node +``` + +```ts +import { FileFinder } from "@ff-labs/fff-node"; + +const finder = FileFinder.create({ basePath: process.cwd(), aiMode: true }); +if (!finder.ok) throw new Error(finder.error); +await finder.value.waitForScan(10_000); + +const files = finder.value.fileSearch("incognito profile", { pageSize: 20 }); +const hits = finder.value.grep("GetOffTheRecordProfile", { + mode: "plain", + smartCase: true, + beforeContext: 1, + afterContext: 1, + classifyDefinitions: true, +}); + +// Run extremely fast glob matching which is significantly (10-100 times) faster than Bun's and Node implementation +const rustFiles = finder.value.glob("**/*.rs", { pageSize: 100 }); + +finder.value.destroy(); +``` + +Every method returns a `Result` (`{ ok: true, value } | { ok: false, error }`). Full type reference: [`packages/fff-node/src/types.ts`](./packages/fff-node/src/types.ts). + +
+ +TypeScript wrapper over the C library for nodejs and bun. Build custom agent tools, CLIs, or IDE integrations on top of FFF. + +
+ +

Rust crate

+ + +### Add the dependency + +FFF is written in Rust, so this is the lowest-overhead way to use it. + +```toml +[dependencies] +fff-search = "0.6" +``` + +Full API documentation: [docs.rs/fff-search](https://docs.rs/fff-search/latest/fff_search/). + +
+ +Native rust crate that is performing all the search. Stable and well documented. + +
+ +

C library

+ + +### Build + +```bash +# Builds only the C cdylib (fastest): +make build-c-lib + +# or directly with cargo: +cargo build --release -p fff-c --features zlob +``` + +> The `zlob` feature (requires the [Zig](https://ziglang.org) toolchain) switches both +> glob matching **and** filesystem traversal to [zlob](https://github.com/dmtrKovalenko/zlob)'s +> native parallel walker. Without it, the default build uses the pure-Rust +> [`ignore`](https://crates.io/crates/ignore) (ripgrep) walker and `globset`. + +The output is a `cdylib` (`libfff_c.so` / `libfff_c.dylib` / `fff_c.dll`). The header lives at [`crates/fff-c/include/fff.h`](./crates/fff-c/include/fff.h). + +Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff.nvim/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages. + +### Install + +```bash +# System-wide (needs sudo): +sudo make install + +# User-local, no sudo: +make install PREFIX=$HOME/.local + +# Staged install for packagers: +make install DESTDIR=/tmp/pkgroot PREFIX=/usr +``` + +Drops `libfff_c.{so,dylib,dll}` into `$(PREFIX)/lib` and the header into `$(PREFIX)/include/fff.h`. Remove with `make uninstall`, which honours the same `PREFIX` and `DESTDIR`. + +Link against it after install: + +```bash +cc my_app.c -lfff_c -o my_app +``` + +Ensure `$(PREFIX)/lib` is on your runtime library search path (`LD_LIBRARY_PATH` on Linux, `DYLD_LIBRARY_PATH` on macOS, or an entry in `/etc/ld.so.conf.d/`). + +### Minimal example + +```c +#include +#include + +int main(void) { + FffResult *res = fff_create_instance( + ".", // base_path + "", // frecency_db_path (empty = default) + "", // history_db_path + false, // use_unsafe_no_lock + true, // enable_mmap_cache + true, // enable_content_indexing + true, // watch + false // ai_mode + ); + if (!res->success) { + fprintf(stderr, "init failed: %s\n", res->error); + fff_free_result(res); + return 1; + } + void *handle = res->handle; + fff_free_result(res); + + // Search + FffResult *search = fff_search(handle, "main.rs", "", 0, 0, 20, 100, 3); + // ... read FffSearchResult from search->handle, then fff_free_search_result() + + fff_destroy(handle); + return 0; +} +``` + +### Versioned options struct (preferred) + +For instance creation use [`FffCreateOptions`](./crates/fff-c/include/fff.h) — a +versioned struct that evolves without ABI breaks. C99 designated +initializers keep call sites readable and zero-init unspecified fields: + +```c +FffResult *res = fff_create_instance_with(&(FffCreateOptions){ + .version = FFF_CREATE_OPTIONS_VERSION, + .base_path = "/path/to/repo", + .ai_mode = true, + .watch = true, + .enable_fs_root_scanning = false, // off by default + .enable_home_dir_scanning = false, // off by default +}); +``` + +### Glob-only search + +`fff_glob` filters indexed files by a single glob pattern, ranks by frecency, +paginates — bypasses the regular query parser entirely. Use this when you +already have a literal glob (`*.rs`, `**/*.test.ts`, `src/**`) and don't want +fuzzy matching layered on top. + +```c +FffResult *res = fff_glob(handle, "**/*.rs", "", 0, 0, 100); +// FffSearchResult in res->handle, free with fff_free_search_result. +``` + +### Notes + +- Every function returning `FffResult*` allocates with Rust's `Box`. Free with `fff_free_result`, do not use malloc's free +- Payloads (search results, grep results, scan progress) have their own dedicated free functions listed in the header. +- C strings returned in the `handle` field (e.g. from `fff_get_base_path`) are freed with `fff_free_string`. + +Source: [`crates/fff-c/`](./crates/fff-c/). + +
+ +Stable C ABI. Bind from C/C++, Zig, Go via cgo, Python via ctypes, or anything with C FFI. + +
+ +

Python bindings

+ + +### Install + +```bash +pip install fff-search +``` + +Or build and install from source: + +```bash +cd packages/fff-python +uv sync --all-extras +uv run maturin develop --release +``` + +### Basic usage + +```python +from fff import FileFinder + +with FileFinder("/path/to/project", watch=False) as finder: + finder.wait_for_scan_blocking(timeout_ms=5000) + + result = finder.search("main") + for item, score in zip(result.items, result.scores): + print(f"{item.relative_path}: {score.total}") + + hits = finder.grep("class Profile", mode="plain", before_context=1, after_context=1) +``` + +### Async usage + +`wait_for_scan` is a coroutine that polls scan status and yields to the event +loop, so it never blocks other tasks. Use `wait_for_scan_blocking` from +synchronous code. + +```python +import asyncio +from fff import FileFinder + +async def main(): + with FileFinder("/path/to/project", watch=False) as finder: + await finder.wait_for_scan(timeout_ms=5000) + result = finder.search("main") + print(result) + +asyncio.run(main()) +``` + +### What you get + +- `search`, `glob`, `directory_search`, `mixed_search` — frecency-ranked fuzzy file/dir search +- `grep` / `multi_grep` — plain, regex, or fuzzy content search with context lines and cursor pagination +- `track_query` / `get_historical_query` — optional frecency and query-history databases +- `reindex`, `refresh_git_status`, `scan_progress`, `health_check` — lifecycle and diagnostics + +Typed result objects (`FileItem`, `Score`, `GrepMatch`, …) with `py.typed` +stubs included. Ships as an `abi3` wheel compatible with Python 3.10+. + +Source: [`packages/fff-python/`](./packages/fff-python/). + +
+ +Native Python bindings built with PyO3. Use them for notebooks, agent scripts, or any Python tool that needs fast file search. + +--- + +## What is FFF and why use it over ripgrep or fzf? + +FFF is a file search library, not a CLI. Ripgrep and fzf are great tools, but they are command-line programs: every call forks a new process, re-reads `.gitignore`, re-stats directories, and rebuilds whatever state it needs in memory before it can answer. That is fine when you grep once from a shell. It is bad when an editor or an AI agent wants to run hundreds of searches per session. + +FFF keeps the index and the file cache resident in one long-lived process and exposes the same Rust core through four thin layers: a native crate (`fff-search`), a C library (`libfff_c`), a Node/Bun SDK (`@ff-labs/fff-node`), and an MCP server. You call `FileFinder.create()` once, then every subsequent search hits warm memory. On a 500k-file Chromium checkout, that is the difference between 3-9 **SECONDS** per ripgrep spawn and sub-10 ms per FFF query. + +Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "\*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema". + +### Why a programmatic API matters + +- No process spawn. Every call stays in-process and avoids the fork, exec, argv parsing, and stdout pipe setup that dominates short `rg` invocations. +- One FS walk, metadata collection, and parse of `.gitignore`. The ignore walker runs once at scan time and the result is reused for every search. +- Results come back as typed objects, not text you have to re-parse. The SDK gives you `{ relativePath, lineNumber, lineContent, gitStatus, totalFrecencyScore, isDefinition, ... }` directly. +- Cursor pagination that survives across calls. Ripgrep has no concept of "page 2 of these matches"; FFF does. +- A long-lived process opens up optimisations that a one-shot CLI cannot apply: warm caches, incremental re-indexing, cross-query frecency, and shared SIMD state. + +### What the core actually does + +- **Frecency-ranked fuzzy matching.** Every indexed file carries an access score and a modification score. Searches rank files you have opened recently and frequently above cold results. This is the same idea as VS Code's recently-opened list, but applied to every search result, not just a sidebar. +- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghen/frizbee)-derived core) that survives dropped characters and reorderings. +- **Content grep with three modes.** Plain literal (SIMD memmem), regex (the Rust `regex` crate), and fuzzy (Smith-Waterman per line). Auto-detects which mode to use from the pattern, falls back to fuzzy when a plain search returns zero hits. +- **Multi-pattern OR search.** SIMD Aho-Corasick for "find any of these 20 identifiers at once", which is faster than regex alternation and a lot faster than 20 separate ripgrep runs. +- **Background file watcher.** The index updates as files change. You never pay for a rescan on the hot path. +- **Git status awareness.** Modified, staged, untracked, and ignored states are cached and returned with every result, so callers can sort or filter them without shelling out to git. The watcher talks to libgit2 directly instead of spawning the `git` CLI. +- **Definition classifier.** A byte-level scanner on the Rust side tags lines that start with `struct`, `fn`, `class`, `def`, `impl`, and friends. + +### Performance choices that matter + +- Efficient memory allocator and memory allocation strategy (see next paragraph). By default we use `mimaloc` +- Parallel multi thread search pipeline that is not contaganted by the orchistration logic +- SIMD first algorithms for everything. Efficinet & non-allocating sorting. +- Platform specific optimizations for FS ([getdents64](https://linux.die.net/man/2/getdents64), NTFS api on windows and others) +- Lightweight on the flight content index for realtime even typo resistant grep +- Memory mapped content cache. We store some of the files in virtual memory (the amount is limited) +- Single contiguous arena storage of string chunks. Significantly reduces the amount of memory to work with and dramatically increases CPU cache hits. + +### Memory allocation + +Yes, fff fundamentally requires more memory than calling a single child process. That is the primary source of the speedup. In practice, alongside one of the most popular file search pickers for Neovim, [fff ends up using less RAM than a burst of ripgrep invocations](https://x.com/neogoose_btw/status/2041606853155811442). + +FFF also keeps a content index, around 360 bytes per indexed file, so roughly 36 MB for a 100k-file repo. Not every file is indexed - binaries, oversized files, and anything not eligible for grep are skipped. If even that footprint is too much, the index can be backed by a memory-mapped file instead of anonymous RAM. + +### What this means in practice + +If you are building an agent, an IDE extension, a pre-commit check, or any long-running tool that searches the same repository many times, calling FFF as a library is dramatically cheaper than shelling out to ripgrep. The tradeoff is real memory: FFF keeps the index in RAM and warms the content cache. On a 14k-file repo that costs about 26 MB resident. On a 500k-file repo like Chromium, expect a few hundred MB. In exchange, every single search is enriched with git status, frecency ranking, file metadata, timestamps of last access and edit and so on. + +If you are running one grep from a terminal, `rg` is still the right tool. If you run dozens of them inside the same process, FFF will pay for itself starting from the second call. If you work on AI agent fff will finish preparation work before your AI will have a chance to call it. + +### How it compares + +- **ripgrep**: FFF uses the same underlying regex engine and more advanced plain text matching algorithms. Stores content index and file tree. Main wins on repeated-search workloads. Loses on "grep once from bash and exit." +- **fzf**: FFF's path search is fuzzy like fzf, but it is also frecency-aware and git-aware, and ships a more typo-tolerant algorithm. fzf is a pure match-and-filter tool; FFF ranks results by how often you actually open them. +- **Telescope / fzf-lua / snacks.picker**: FFF ships its own Neovim picker with the same ranking the MCP server and SDK use. The picker is optional; the core is the same. +- **Tantivy or other full-text search engines**: different class of tool. Tantivy indexes documents for query-time scoring at scale. FFF is scoped to one repository and optimised for sub-10 ms response. It does not persist an inverted index on disk. + +--- + +## Repository layout + +- `crates/fff-search`, `crates/fff-grep`, `crates/fff-query-parser` - Rust core. +- `crates/fff-c` - C FFI used by every language binding. +- `crates/fff-nvim` - Lua/mlua bindings for the Neovim plugin. +- `crates/fff-mcp` - MCP server binary. +- `packages/fff-node` - Node.js SDK (`@ff-labs/fff-node`). +- `packages/fff-bun` - Bun SDK (`@ff-labs/fff-bun`). +- `packages/pi-fff` - pi extension (`@ff-labs/pi-fff`). +- `lua/` - Neovim-side plugin code. + +## Contributing + +Bug reports and pull requests welcome. Agentic coding tools are welcome to be used, but human review is mandatory. + +## License + +[MIT](./LICENSE) & open source forever. diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..24d7866 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`dmtrKovalenko/fff.nvim` +- 原始仓库：https://github.com/dmtrKovalenko/fff.nvim +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/_typos.toml b/_typos.toml new file mode 100644 index 0000000..6c280a3 --- /dev/null +++ b/_typos.toml @@ -0,0 +1,26 @@ +[files] +extend-exclude = ["/CHANGELOG.md", "data/filetypes/base.lua"] + +[default.extend-words] +noice = "noice" +fo = "fo" +ba = "ba" +ue = "ue" +# file extensions that look like typos +thm = "thm" +# some typos we use for tests +comparsion = "comparsion" +modfiers = "modfiers" +shcema = "shcema" + +[default] +extend-ignore-re = [ + # git commit hashes in CHANGELOG.md + "\\[[0-9a-f]{7}\\]", + # Line ignore with trailing # spellchecker:disable-line + "(?Rm)^.*(#|--|//)\\s*spellchecker:disable-line$", + # Line block with # spellchecker: + "(?s)(#|--|//|)\\s*spellchecker:off.*?\\n\\s*(#|--|//)\\s*spellchecker:on", + # Vim doc formatting + "vim:.+:" +] diff --git a/assets/logo-dark.png b/assets/logo-dark.png new file mode 100644 index 0000000..57e2b76 Binary files /dev/null and b/assets/logo-dark.png differ diff --git a/assets/logo-light.png b/assets/logo-light.png new file mode 100644 index 0000000..e3d27ac Binary files /dev/null and b/assets/logo-light.png differ diff --git a/assets/logo-orange.png b/assets/logo-orange.png new file mode 100644 index 0000000..1b3c6c8 Binary files /dev/null and b/assets/logo-orange.png differ diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..2103849 --- /dev/null +++ b/biome.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.7/schema.json", + "files": { + "includes": ["packages/**/*.ts", "!packages/*/dist"], + "ignoreUnknown": true + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 90 + }, + "javascript": { + "formatter": { + "quoteStyle": "double", + "trailingCommas": "all", + "semicolons": "always" + } + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": { + "noNonNullAssertion": "off" + }, + "suspicious": { + "noExplicitAny": "off" + }, + "complexity": { + "noForEach": "off" + } + } + } +} diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..568f126 --- /dev/null +++ b/bun.lock @@ -0,0 +1,417 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "devDependencies": { + "@biomejs/biome": "^2.4.4", + }, + }, + "packages/fff-bun": { + "name": "@ff-labs/fff-bun", + "version": "0.1.37", + "bin": { + "fff-demo": "./examples/search.ts", + "fff-grep": "./examples/grep.ts", + }, + "devDependencies": { + "@types/bun": "^1.3.8", + "typescript": "^5.0.0", + }, + "optionalDependencies": { + "@ff-labs/fff-bin-darwin-arm64": "0.0.0", + "@ff-labs/fff-bin-darwin-x64": "0.0.0", + "@ff-labs/fff-bin-linux-arm64-gnu": "0.0.0", + "@ff-labs/fff-bin-linux-arm64-musl": "0.0.0", + "@ff-labs/fff-bin-linux-x64-gnu": "0.0.0", + "@ff-labs/fff-bin-linux-x64-musl": "0.0.0", + "@ff-labs/fff-bin-win32-arm64": "0.0.0", + "@ff-labs/fff-bin-win32-x64": "0.0.0", + }, + }, + "packages/fff-node": { + "name": "@ff-labs/fff-node", + "version": "0.1.37", + "dependencies": { + "ffi-rs": "^1.0.0", + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.0.0", + }, + "optionalDependencies": { + "@ff-labs/fff-bin-darwin-arm64": "0.0.0", + "@ff-labs/fff-bin-darwin-x64": "0.0.0", + "@ff-labs/fff-bin-linux-arm64-gnu": "0.0.0", + "@ff-labs/fff-bin-linux-arm64-musl": "0.0.0", + "@ff-labs/fff-bin-linux-x64-gnu": "0.0.0", + "@ff-labs/fff-bin-linux-x64-musl": "0.0.0", + "@ff-labs/fff-bin-win32-arm64": "0.0.0", + "@ff-labs/fff-bin-win32-x64": "0.0.0", + }, + }, + "packages/pi-fff": { + "name": "@ff-labs/pi-fff", + "version": "0.6.0", + "dependencies": { + "@ff-labs/fff-node": "*", + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.0.0", + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*", + "@sinclair/typebox": "*", + }, + }, + }, + "packages": { + "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.91.1", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw=="], + + "@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="], + + "@aws-crypto/sha256-browser": ["@aws-crypto/sha256-browser@5.2.0", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw=="], + + "@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="], + + "@aws-crypto/supports-web-crypto": ["@aws-crypto/supports-web-crypto@5.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg=="], + + "@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="], + + "@aws-sdk/client-bedrock-runtime": ["@aws-sdk/client-bedrock-runtime@3.1048.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.11", "@aws-sdk/credential-provider-node": "^3.972.42", "@aws-sdk/eventstream-handler-node": "^3.972.16", "@aws-sdk/middleware-eventstream": "^3.972.12", "@aws-sdk/middleware-websocket": "^3.972.19", "@aws-sdk/token-providers": "3.1048.0", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ=="], + + "@aws-sdk/core": ["@aws-sdk/core@3.974.18", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@aws-sdk/xml-builder": "^3.972.28", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.6", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-JDYCPI0j7zGrzXTDFsLB346cxss7J/AxH7+O0MzWlqppJBEyB9Qe6TQXRL6iwLUo/xZkNv9KFmBL2hqElmwW0g=="], + + "@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.44", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-3hKJVrZ7bqXzDAXCQp+OaQ1ASN+vWstaNuEH418wQVl//cRZhqhfR9Bjk1qIWmgUGe8/D3gdO73PgidRj378EQ=="], + + "@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.46", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/node-http-handler": "^4.7.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-VhwC9pGAZHhiQ2xSViyOPDFqvr9aRxGCAXZtADsUhU3R65nad7y//CwynE6mQnWNR+suRlqE79W36IVayL+m1g=="], + + "@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.972.50", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/credential-provider-env": "^3.972.44", "@aws-sdk/credential-provider-http": "^3.972.46", "@aws-sdk/credential-provider-login": "^3.972.49", "@aws-sdk/credential-provider-process": "^3.972.44", "@aws-sdk/credential-provider-sso": "^3.972.49", "@aws-sdk/credential-provider-web-identity": "^3.972.49", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/credential-provider-imds": "^4.3.7", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-09Xi6ovxiK42+De/qBGF71sT5F2bWgYM+1fFyDwSOpy1xpsQ5R/naIu7MVDpH6Dic36QNc8dAv4KADtMGK2JYg=="], + + "@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-EfJF/1Fh9mI4pZyoheU2RY9xUhTcugIZNkD63+orXMkYj/QXacJNbKVDUK90Yv5hE+aX+rt9J/EZ9Qr3vKOa7g=="], + + "@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.52", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.44", "@aws-sdk/credential-provider-http": "^3.972.46", "@aws-sdk/credential-provider-ini": "^3.972.50", "@aws-sdk/credential-provider-process": "^3.972.44", "@aws-sdk/credential-provider-sso": "^3.972.49", "@aws-sdk/credential-provider-web-identity": "^3.972.49", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/credential-provider-imds": "^4.3.7", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-7QX+PbyiWBEOVipJq8Nke/TqXT6lAPLE7fvTaopa39/IVWuLfS+Fzdy71sZJONf/mLGgmtj6aU17+REw3+aRrw=="], + + "@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.44", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-V+UUhZpRP7QDRhi+qgBDisM9tUBnYmMje8Bk77A6MZsfeGeGdMsQXmaHP1CDYFcept0o/Rz5g2Y0TMeVlG9dzg=="], + + "@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/token-providers": "3.1063.0", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-9QqOYGuh5tZ76OzaT68kwI78AH+5lS/uZGGvkfxb3fc8FzRrIz2jOufNTliEBEeSAwmgK2rWLNsK+IB3zbtNPA=="], + + "@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-IYx1lN38MnnPXv+NBLpuATu0cZakbZ321TAfjW+aVkw7HIJF38YnEwdeEO55MSl3pl7hIX1IvvnD6EmnAzmAJw=="], + + "@aws-sdk/eventstream-handler-node": ["@aws-sdk/eventstream-handler-node@3.972.20", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-qr/S1iFCDIXlZwlZPaCqjKcHbJFr9scIFUhbh2+SrwPXZvRhyOUWjVDJpp8xoU4qrrMR0PqK1Yw5C2sSj7xAyw=="], + + "@aws-sdk/middleware-eventstream": ["@aws-sdk/middleware-eventstream@3.972.16", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-KR2Gdui/QLbkdG9FxW3vk/vIa8KiDP5vQBNERo7MmlPHjn23GXJ53Cq5P/ok7/ALbTUiYZ78DiBHoDcvzPWvgQ=="], + + "@aws-sdk/middleware-websocket": ["@aws-sdk/middleware-websocket@3.972.26", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-foM3KvxGBHY9lRIm6C9JJJ5haodtXfJPPgJQcv5/c4A2pN4I7tlnOjh1o2d8Il1Y/j6GWOw3YeIYc2/VYjtGVQ=="], + + "@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.17", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.18", "@aws-sdk/signature-v4-multi-region": "^3.996.32", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/node-http-handler": "^4.7.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-lDRgraoTfKRawUyc176Ow93mrNrOho/x+EoK4C+lKU+vKkHWhNhzvSMVAx0WEJUJoeQxxDN5ZdKMfiGEyNejig=="], + + "@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.32", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-llvApLcsWtmRFhG2wT3WIp1CmDeRaIYutqty1ZZXoMzK7TiJ6MOLOimk9eXUS8PwgG4ew4pa4QAbt0lfhn++1w=="], + + "@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1048.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.11", "@aws-sdk/nested-clients": "^3.997.9", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA=="], + + "@aws-sdk/types": ["@aws-sdk/types@3.973.11", "", { "dependencies": { "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-YjS0qFuECClRh4qhEyW8XagW0fwEPBeZ1cfsW/gU73Kh/ExFILxbzxOfPCmzF/2DwEvhvsHYt0b0qnvStwKYrg=="], + + "@aws-sdk/util-locate-window": ["@aws-sdk/util-locate-window@3.965.6", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-ZfHjfwSzeXj+Lg9AK5ZNmeDkXev6V+w2tn1t4kgDdRtUaRCthepTQiFwbD06EF9oNGH4LaLg+Mb6U16Ypv5bSw=="], + + "@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.28", "", { "dependencies": { "@smithy/types": "^4.14.3", "fast-xml-parser": "5.7.3", "tslib": "^2.6.2" } }, "sha512-lI/l3c/vPvsxmspzV63NfS3x9q4CkMmdhJy4QiM+NThAufVkDvi/PZZQ6xETnICL0UD7jI808pY83gllf86RFg=="], + + "@aws/lambda-invoke-store": ["@aws/lambda-invoke-store@0.2.4", "", {}, "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ=="], + + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], + + "@biomejs/biome": ["@biomejs/biome@2.4.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.4", "@biomejs/cli-darwin-x64": "2.4.4", "@biomejs/cli-linux-arm64": "2.4.4", "@biomejs/cli-linux-arm64-musl": "2.4.4", "@biomejs/cli-linux-x64": "2.4.4", "@biomejs/cli-linux-x64-musl": "2.4.4", "@biomejs/cli-win32-arm64": "2.4.4", "@biomejs/cli-win32-x64": "2.4.4" }, "bin": { "biome": "bin/biome" } }, "sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q=="], + + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A=="], + + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dh1a/+W+SUCXhEdL7TiX3ArPTFCQKJTI1mGncZNWfO+6suk+gYA4lNyJcBB+pwvF49uw0pEbUS49BgYOY4hzUg=="], + + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-V/NFfbWhsUU6w+m5WYbBenlEAz8eYnSqRMDMAW3K+3v0tYVkNyZn8VU0XPxk/lOqNXLSCCrV7FmV/u3SjCBShg=="], + + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-+sPAXq3bxmFwhVFJnSwkSF5Rw2ZAJMH3MF6C9IveAEOdSpgajPhoQhbbAK12SehN9j2QrHpk4J/cHsa/HqWaYQ=="], + + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-R4+ZCDtG9kHArasyBO+UBD6jr/FcFCTH8QkNTOCu0pRJzCWyWC4EtZa2AmUZB5h3e0jD7bRV2KvrENcf8rndBg=="], + + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-gGvFTGpOIQDb5CQ2VC0n9Z2UEqlP46c4aNgHmAMytYieTGEcfqhfCFnhs6xjt0S3igE6q5GLuIXtdQt3Izok+g=="], + + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-trzCqM7x+Gn832zZHgr28JoYagQNX4CZkUZhMUac2YxvvyDRLJDrb5m9IA7CaZLlX6lTQmADVfLEKP1et1Ma4Q=="], + + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.4", "", { "os": "win32", "cpu": "x64" }, "sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A=="], + + "@earendil-works/pi-agent-core": ["@earendil-works/pi-agent-core@0.78.1", "", { "dependencies": { "@earendil-works/pi-ai": "^0.78.1", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" } }, "sha512-oPwVRkkAvyKPWyM7E4k+EaTNmynbYn7ZLG/LBh9BUnMNb2gvpMp+VQ420R6JCJ20uogSqrHnWTyosSa/rU8lVw=="], + + "@earendil-works/pi-ai": ["@earendil-works/pi-ai@0.78.1", "", { "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.1", "@smithy/node-http-handler": "4.7.3", "http-proxy-agent": "7.0.2", "https-proxy-agent": "7.0.6", "openai": "6.26.0", "partial-json": "0.1.7", "typebox": "1.1.38" }, "bin": { "pi-ai": "dist/cli.js" } }, "sha512-CM2pkTs1iupG/maw381lC9Q/Y/aQaMGK7GILc28ttImD0ci3LDwKroDsGkWbly5JIy3iqxdRxB9JlG7vvzCzTg=="], + + "@earendil-works/pi-coding-agent": ["@earendil-works/pi-coding-agent@0.78.1", "", { "dependencies": { "@earendil-works/pi-agent-core": "^0.78.1", "@earendil-works/pi-ai": "^0.78.1", "@earendil-works/pi-tui": "^0.78.1", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", "diff": "8.0.4", "glob": "13.0.6", "highlight.js": "10.7.3", "hosted-git-info": "9.0.3", "ignore": "7.0.5", "jiti": "2.7.0", "minimatch": "10.2.5", "proper-lockfile": "4.1.2", "typebox": "1.1.38", "undici": "8.3.0", "yaml": "2.9.0" }, "optionalDependencies": { "@mariozechner/clipboard": "0.3.9" }, "bin": { "pi": "dist/cli.js" } }, "sha512-Syjf6Ib8UoY5t9ZdKjp0BRrQZuFkFBc8j2KEU9zG/ZnmYPcAxYeioofdv2Q3MEXnHEX2U8sKQptkSnJIdMsd0g=="], + + "@earendil-works/pi-tui": ["@earendil-works/pi-tui@0.78.1", "", { "dependencies": { "get-east-asian-width": "1.6.0", "marked": "15.0.12" } }, "sha512-07GVQo/38a0yvIPlWDr3RJn1B8gk3ZuIX9h2oIQ+Biyu3JN0KppWmgWHfaWRydQgse5JtC++KDw5MWaIRnV0mw=="], + + "@ff-labs/fff-bun": ["@ff-labs/fff-bun@workspace:packages/fff-bun"], + + "@ff-labs/fff-node": ["@ff-labs/fff-node@workspace:packages/fff-node"], + + "@ff-labs/pi-fff": ["@ff-labs/pi-fff@workspace:packages/pi-fff"], + + "@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="], + + "@mariozechner/clipboard": ["@mariozechner/clipboard@0.3.9", "", { "optionalDependencies": { "@mariozechner/clipboard-darwin-arm64": "0.3.9", "@mariozechner/clipboard-darwin-universal": "0.3.9", "@mariozechner/clipboard-darwin-x64": "0.3.9", "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-musl": "0.3.9", "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" } }, "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA=="], + + "@mariozechner/clipboard-darwin-arm64": ["@mariozechner/clipboard-darwin-arm64@0.3.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ=="], + + "@mariozechner/clipboard-darwin-universal": ["@mariozechner/clipboard-darwin-universal@0.3.9", "", { "os": "darwin" }, "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ=="], + + "@mariozechner/clipboard-darwin-x64": ["@mariozechner/clipboard-darwin-x64@0.3.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg=="], + + "@mariozechner/clipboard-linux-arm64-gnu": ["@mariozechner/clipboard-linux-arm64-gnu@0.3.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw=="], + + "@mariozechner/clipboard-linux-arm64-musl": ["@mariozechner/clipboard-linux-arm64-musl@0.3.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ=="], + + "@mariozechner/clipboard-linux-riscv64-gnu": ["@mariozechner/clipboard-linux-riscv64-gnu@0.3.9", "", { "os": "linux", "cpu": "none" }, "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw=="], + + "@mariozechner/clipboard-linux-x64-gnu": ["@mariozechner/clipboard-linux-x64-gnu@0.3.9", "", { "os": "linux", "cpu": "x64" }, "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw=="], + + "@mariozechner/clipboard-linux-x64-musl": ["@mariozechner/clipboard-linux-x64-musl@0.3.9", "", { "os": "linux", "cpu": "x64" }, "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ=="], + + "@mariozechner/clipboard-win32-arm64-msvc": ["@mariozechner/clipboard-win32-arm64-msvc@0.3.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ=="], + + "@mariozechner/clipboard-win32-x64-msvc": ["@mariozechner/clipboard-win32-x64-msvc@0.3.9", "", { "os": "win32", "cpu": "x64" }, "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA=="], + + "@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="], + + "@nodable/entities": ["@nodable/entities@2.1.1", "", {}, "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg=="], + + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.2", "", {}, "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], + + "@silvia-odwyer/photon-node": ["@silvia-odwyer/photon-node@0.3.4", "", {}, "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA=="], + + "@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="], + + "@smithy/core": ["@smithy/core@3.24.6", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-wBXDRup6UU97VKyaiRo8AssnfStPtG0oAAfpq/bC0a1YYau8pM86YB4kM6ccoVi1mS8l/UHbn9oDM+7uozr/ug=="], + + "@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.3.8", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-5cAM+KZC02sTqDt6NaLXyu50M/GNMd1eTzDVR8Lb0BBsVtu7RWHo47VPPEEv1vt3Yub6uzr+M5FHC+GtoT0USg=="], + + "@smithy/fetch-http-handler": ["@smithy/fetch-http-handler@5.4.6", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-FEwEYJ1jlBKdhe9TPzfghEi1bP55ZeEImlDkEa62bBBYzUcnB6RUCyuiS2mqKt6ZVjUbBgcNhzfIctH+Hevx9g=="], + + "@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="], + + "@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.3", "", { "dependencies": { "@smithy/core": "^3.24.3", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA=="], + + "@smithy/signature-v4": ["@smithy/signature-v4@5.4.6", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-Ojg4B6oIDlIr1R86xCDJt1zJWnYa0VINmqdjfe9qxWjdRivHalZ3iSlQgVqYbW0MdpFOC5XfHEWsnbmdnpIILQ=="], + + "@smithy/types": ["@smithy/types@4.14.3", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-YupL0ZWmFtJexUN2cHzkvvF/b9pKrtAIfT1o7/oY/Ppu8IYeZ+lDPM5vZdQJaSeA132dJCqojjGC9NhXeF71VQ=="], + + "@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="], + + "@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], + + "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], + + "@types/node": ["@types/node@22.19.15", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg=="], + + "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], + + "@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="], + + "@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="], + + "@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="], + + "@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="], + + "@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="], + + "@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="], + + "@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="], + + "@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="], + + "@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="], + + "@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="], + + "@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="], + + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], + + "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + + "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="], + + "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], + + "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], + + "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="], + + "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="], + + "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], + + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + + "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "diff": ["diff@8.0.4", "", {}, "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw=="], + + "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], + + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + + "fast-xml-builder": ["fast-xml-builder@1.2.0", "", { "dependencies": { "path-expression-matcher": "^1.5.0", "xml-naming": "^0.1.0" } }, "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q=="], + + "fast-xml-parser": ["fast-xml-parser@5.7.3", "", { "dependencies": { "@nodable/entities": "^2.1.0", "fast-xml-builder": "^1.1.7", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg=="], + + "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], + + "ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="], + + "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], + + "gaxios": ["gaxios@7.1.5", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg=="], + + "gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="], + + "get-east-asian-width": ["get-east-asian-width@1.6.0", "", {}, "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA=="], + + "glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="], + + "google-auth-library": ["google-auth-library@10.7.0", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-QpTAbNJ36TliZLx3TTtahR8HG0hN9RllL1e3FymOvQSIKK8JmgV58H924ub2wa2DsS3ANjjP1Aw1N+Ramc8hqQ=="], + + "google-logging-utils": ["google-logging-utils@1.1.3", "", {}, "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA=="], + + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + + "highlight.js": ["highlight.js@10.7.3", "", {}, "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A=="], + + "hosted-git-info": ["hosted-git-info@9.0.3", "", { "dependencies": { "lru-cache": "^11.1.0" } }, "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg=="], + + "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="], + + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + + "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], + + "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], + + "json-schema-to-ts": ["json-schema-to-ts@3.1.1", "", { "dependencies": { "@babel/runtime": "^7.18.3", "ts-algebra": "^2.0.0" } }, "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g=="], + + "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], + + "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + + "lru-cache": ["lru-cache@11.5.1", "", {}, "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A=="], + + "marked": ["marked@15.0.12", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA=="], + + "minimatch": ["minimatch@10.2.5", "", { "dependencies": { "brace-expansion": "^5.0.5" } }, "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg=="], + + "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], + + "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + + "openai": ["openai@6.26.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA=="], + + "p-retry": ["p-retry@4.6.2", "", { "dependencies": { "@types/retry": "0.12.0", "retry": "^0.13.1" } }, "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ=="], + + "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="], + + "path-expression-matcher": ["path-expression-matcher@1.5.0", "", {}, "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ=="], + + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="], + + "proper-lockfile": ["proper-lockfile@4.1.2", "", { "dependencies": { "graceful-fs": "^4.2.4", "retry": "^0.12.0", "signal-exit": "^3.0.2" } }, "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA=="], + + "protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="], + + "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], + + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + + "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], + + "strnum": ["strnum@2.3.0", "", {}, "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q=="], + + "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "typebox": ["typebox@1.1.38", "", {}, "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici": ["undici@8.3.0", "", {}, "sha512-TkUDgb6tl7KOGZ+7e8E3d2FYgUQgF6z5YypqjWmixVQSQERFcVrVg0ySADm2LVLRh5ljAaHTCR5Fmz3Q34rB7Q=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], + + "xml-naming": ["xml-naming@0.1.0", "", {}, "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw=="], + + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], + + "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], + + "@aws-sdk/credential-provider-http/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.7", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-ZAFvHXrEk6K180EVhmZVg8GU5pUH5BSFqRs27JW3j1qEFx9YyYwWFx17x/MHcjALYimGAji7qEOlF1++be+G5A=="], + + "@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1063.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-nYDaWWdzjKiDP5xj8k4oUgcYd4WPgzfAOgdU5vJsaqH/07Dfvm7ffisHCFJ+NEl7kUC9JEIUxh0kznvenbo3NQ=="], + + "@aws-sdk/nested-clients/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.7", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-ZAFvHXrEk6K180EVhmZVg8GU5pUH5BSFqRs27JW3j1qEFx9YyYwWFx17x/MHcjALYimGAji7qEOlF1++be+G5A=="], + + "bun-types/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="], + + "p-retry/retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], + + "protobufjs/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="], + + "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + + "protobufjs/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + } +} diff --git a/chart.png b/chart.png new file mode 100644 index 0000000..00ffea1 Binary files /dev/null and b/chart.png differ diff --git a/crates/fff-c/Cargo.toml b/crates/fff-c/Cargo.toml new file mode 100644 index 0000000..72aea2c --- /dev/null +++ b/crates/fff-c/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "fff-c" +version = "0.9.6" +edition = "2024" +description = "Raw C api of FFF file finder" +license = "MIT" + +[lib] +crate-type = ["cdylib"] + +[features] +default = ["ripgrep"] # use ripgrep base crates to avoid requiring zig for rust crate +ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"] +zlob = ["fff/zlob", "fff-query-parser/zlob"] + +[dependencies] +git2.workspace = true + +fff = { package = "fff-search", path = "../fff-core", version = "0.9.6", default-features = false } +fff-query-parser = { path = "../fff-query-parser", version = "0.9.6", default-features = false } +serde_json = "1.0" diff --git a/crates/fff-c/cbindgen.toml b/crates/fff-c/cbindgen.toml new file mode 100644 index 0000000..86c67f9 --- /dev/null +++ b/crates/fff-c/cbindgen.toml @@ -0,0 +1,33 @@ +language = "C" +header = "/* Generated by cbindgen — do not edit manually. */" +include_guard = "FFF_C_H" +include_version = true +no_includes = true +sys_includes = ["stdint.h", "stdbool.h", "stddef.h"] + +[export] +include = [ + "FffResult", + "FffSearchResult", "FffFileItem", "FffScore", "FffLocation", + "FffGrepResult", "FffGrepMatch", "FffMatchRange", + "FffScanProgress", +] + +[export.rename] +"FffResult" = "FffResult" +"FffSearchResult" = "FffSearchResult" +"FffFileItem" = "FffFileItem" +"FffScore" = "FffScore" +"FffLocation" = "FffLocation" +"FffGrepResult" = "FffGrepResult" +"FffGrepMatch" = "FffGrepMatch" +"FffMatchRange" = "FffMatchRange" +"FffScanProgress" = "FffScanProgress" + +[fn] +sort_by = "None" +# Translate `#[deprecated]` on extern "C" fns into a real C compiler +# attribute so callers get a warning when they use a removed/legacy entry. +# `{}` is substituted with the Rust deprecation note as a C string literal +# (already quoted) — do not wrap in extra quotes. +deprecated_with_note = "__attribute__((deprecated({})))" diff --git a/crates/fff-c/include/fff.h b/crates/fff-c/include/fff.h new file mode 100644 index 0000000..8bc4205 --- /dev/null +++ b/crates/fff-c/include/fff.h @@ -0,0 +1,1353 @@ +/* Generated by cbindgen — do not edit manually. */ + +#ifndef FFF_C_H +#define FFF_C_H + +/* Generated with cbindgen:0.29.2 */ + +#include +#include +#include + +/** + * Current used version of [`FffCreateOptions`]. + */ +#define FFF_CREATE_OPTIONS_VERSION 2 + +/** + * Result envelope returned by all `fff_*` functions. + * + * Heap-allocated. The caller must free it with `fff_free_result`. Calling `fff_free_result` + * **does not** deallocate the underlying `handle` pointer. It needs to be cleaned separately. + * see (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`, `fff_free_string`, etc.). + * + * Depending on the function, the payload is delivered through different fields: + * + * | Function | Payload field | Type | + * |----------------------------|---------------|-------------------------------| + * | `fff_create_instance` | `handle` | opaque instance pointer | + * | `fff_search` | `handle` | `*mut FffSearchResult` | + * | `fff_live_grep` | `handle` | `*mut FffGrepResult` | + * | `fff_multi_grep` | `handle` | `*mut FffGrepResult` | + * | `fff_get_scan_progress` | `handle` | `*mut FffScanProgress` | + * | `fff_health_check` | `handle` | `*mut c_char` (JSON string) | + * | `fff_get_historical_query` | `handle` | `*mut c_char` (string or null)| + * | `fff_wait_for_scan` | `int_value` | 1 = completed, 0 = timed out | + * | `fff_track_query` | `int_value` | 1 = success, 0 = failure | + * | `fff_refresh_git_status` | `int_value` | number of files updated | + * | `fff_scan_files` | (none) | success flag only | + * | `fff_restart_index` | (none) | success flag only | + * + * On failure, `success` is false and `error` contains the message. + */ +typedef struct FffResult { + /** + * Whether the operation succeeded. + */ + bool success; + /** + * Error message on failure. Null on success. + */ + char *error; + /** + * Opaque pointer payload. May be null. + */ + void *handle; + /** + * Integer payload for simple return values (bool as 0/1, counts, etc.). + */ + int64_t int_value; +} FffResult; + +/** + * Options for `fff_create_instance_with`. + * + * Versioned struct: you populate the struct at your call level, we guarantee that + * the version is stable across the version changes, new fields only appended! + */ +typedef struct FffCreateOptions { + /** + * Set to [`FFF_CREATE_OPTIONS_VERSION`] when allocating. Used by the + * library to determine which trailing fields are populated. + */ + uint32_t version; + /** + * Directory to index (required, non-NULL). + */ + const char *base_path; + /** + * Frecency LMDB database path. NULL/empty to skip frecency tracking. + */ + const char *frecency_db_path; + /** + * Query history LMDB database path. NULL/empty to skip query tracking. + */ + const char *history_db_path; + /** + * Pre-populate mmap caches for top-frecency files after the initial scan. + */ + bool enable_mmap_cache; + /** + * Build content index after the initial scan for faster grep. + */ + bool enable_content_indexing; + /** + * Start a background file-system watcher for live updates. + */ + bool watch; + /** + * Enable AI-agent optimizations. + */ + bool ai_mode; + /** + * Tracing log file path. NULL/empty to skip log init. + */ + const char *log_file_path; + /** + * Log level: `"trace" | "debug" | "info" | "warn" | "error"`. + * NULL/empty defaults to `"info"`. Ignored when `log_file_path` is unset. + */ + const char *log_level; + /** + * Content cache file-count cap. 0 = auto. + */ + uint64_t cache_budget_max_files; + /** + * Content cache byte cap. 0 = auto. + */ + uint64_t cache_budget_max_bytes; + /** + * Per-file byte cap inside the content cache. 0 = auto. + */ + uint64_t cache_budget_max_file_size; + /** + * Allow indexing the filesystem root (`/`). Off by default — root is + * rarely the intended target and floods the watcher with churn. + */ + bool enable_fs_root_scanning; + /** + * Allow indexing the user's home directory. Same trade-off as + * `enable_fs_root_scanning`. + */ + bool enable_home_dir_scanning; + /** + * Follow symlinks during scan and watcher walks. Off by default — + * enabling this without external loop protection can wedge the watcher + * on cyclic symlink graphs. Caller is responsible for the trade-off. + */ + bool follow_symlinks; +} FffCreateOptions; + +/** + * A file item returned by `fff_search`. + * + * All string fields are heap-allocated and owned by the parent `FffSearchResult`. + * Free the entire result with `fff_free_search_result`. + */ +typedef struct FffFileItem { + char *relative_path; + char *file_name; + char *git_status; + uint64_t size; + uint64_t modified; + int64_t access_frecency_score; + int64_t modification_frecency_score; + int64_t total_frecency_score; + bool is_binary; +} FffFileItem; + +/** + * Score breakdown for a search result. + */ +typedef struct FffScore { + int32_t total; + int32_t base_score; + int32_t filename_bonus; + int32_t special_filename_bonus; + int32_t frecency_boost; + int32_t distance_penalty; + int32_t current_file_penalty; + int32_t combo_match_boost; + int32_t path_alignment_bonus; + bool exact_match; + char *match_type; +} FffScore; + +/** + * Location parsed from a query string (e.g. `"file.ts:42:10"`). + * + * `tag` encodes the variant: + * 0 = no location, + * 1 = line only (`line` is set), + * 2 = position (`line` + `col`), + * 3 = range (`line`/`col` = start, `end_line`/`end_col` = end). + */ +typedef struct FffLocation { + uint8_t tag; + int32_t line; + int32_t col; + int32_t end_line; + int32_t end_col; +} FffLocation; + +/** + * Search result returned by `fff_search`. + * + * The caller must free this with `fff_free_search_result`. + */ +typedef struct FffSearchResult { + /** + * Pointer to a heap-allocated array of `FffFileItem` (length = `count`). + */ + struct FffFileItem *items; + /** + * Pointer to a heap-allocated array of `FffScore` (length = `count`). + */ + struct FffScore *scores; + /** + * Number of items/scores in the arrays. + */ + uint32_t count; + /** + * Total number of files that matched the query. + */ + uint32_t total_matched; + /** + * Total number of indexed files. + */ + uint32_t total_files; + /** + * Location parsed from the query string. + */ + struct FffLocation location; +} FffSearchResult; + +/** + * A byte range within a matched line, used for highlighting. + */ +typedef struct FffMatchRange { + uint32_t start; + uint32_t end; +} FffMatchRange; + +/** + * A single grep match with file and line information. + * + * All string fields and arrays are heap-allocated. Free the parent + * `FffGrepResult` with `fff_free_grep_result` to release everything. + */ +typedef struct FffGrepMatch { + char *relative_path; + char *file_name; + char *git_status; + char *line_content; + struct FffMatchRange *match_ranges; + char **context_before; + char **context_after; + uint64_t size; + uint64_t modified; + int64_t total_frecency_score; + int64_t access_frecency_score; + int64_t modification_frecency_score; + uint64_t line_number; + uint64_t byte_offset; + uint32_t col; + uint32_t match_ranges_count; + uint32_t context_before_count; + uint32_t context_after_count; + uint16_t fuzzy_score; + bool has_fuzzy_score; + bool is_binary; + bool is_definition; +} FffGrepMatch; + +/** + * Grep result returned by `fff_live_grep` and `fff_multi_grep`. + * + * The caller must free this with `fff_free_grep_result`. + */ +typedef struct FffGrepResult { + /** + * Pointer to a heap-allocated array of `FffGrepMatch` (length = `count`). + */ + struct FffGrepMatch *items; + /** + * Number of matches in the `items` array. + */ + uint32_t count; + /** + * Total number of matches (always equal to `count`). + */ + uint32_t total_matched; + /** + * Number of files actually opened and searched in this call. + */ + uint32_t total_files_searched; + /** + * Total number of indexed files (before any filtering). + */ + uint32_t total_files; + /** + * Number of files eligible for search after filtering. + */ + uint32_t filtered_file_count; + /** + * File offset for the next page. 0 if all files have been searched. + */ + uint32_t next_file_offset; + /** + * Regex compilation error when falling back to literal matching. Null if none. + */ + char *regex_fallback_error; +} FffGrepResult; + +/** + * Scan progress returned by `fff_get_scan_progress`. + * The caller must free this with `fff_free_scan_progress`. + */ +typedef struct FffScanProgress { + uint64_t scanned_files_count; + bool is_scanning; + bool is_watcher_ready; + bool is_warmup_complete; +} FffScanProgress; + +/** + * A directory item returned by `fff_search_directories`. + * + * All string fields are heap-allocated and owned by the parent `FffDirSearchResult`. + * Free the entire result with `fff_free_dir_search_result`. + */ +typedef struct FffDirItem { + char *relative_path; + char *dir_name; + int32_t max_access_frecency; +} FffDirItem; + +/** + * Directory search result returned by `fff_search_directories`. + * + * The caller must free this with `fff_free_dir_search_result`. + */ +typedef struct FffDirSearchResult { + /** + * Pointer to a heap-allocated array of `FffDirItem` (length = `count`). + */ + struct FffDirItem *items; + /** + * Pointer to a heap-allocated array of `FffScore` (length = `count`). + */ + struct FffScore *scores; + /** + * Number of items/scores in the arrays. + */ + uint32_t count; + /** + * Total number of directories that matched the query. + */ + uint32_t total_matched; + /** + * Total number of indexed directories. + */ + uint32_t total_dirs; +} FffDirSearchResult; + +/** + * A single item in a mixed (files + directories) search result. + * + * `item_type`: 0 = file, 1 = directory. + * All string fields are heap-allocated and owned by the parent `FffMixedSearchResult`. + */ +typedef struct FffMixedItem { + /** + * 0 = file, 1 = directory. + */ + uint8_t item_type; + char *relative_path; + /** + * Filename for files, last directory segment for directories. + */ + char *display_name; + char *git_status; + uint64_t size; + uint64_t modified; + /** + * The access frecency score for files, or max access frecency among all the immediate + * children for directories. + */ + int64_t access_frecency_score; + /** + * Always 0 for directories + */ + int64_t modification_frecency_score; + /** + * Always 0 for directories + */ + int64_t total_frecency_score; + /** + * Always 0 for directories + */ + bool is_binary; +} FffMixedItem; + +/** + * Mixed search result returned by `fff_search_mixed`. + * + * The caller must free this with `fff_free_mixed_search_result`. + */ +typedef struct FffMixedSearchResult { + /** + * Pointer to a heap-allocated array of `FffMixedItem` (length = `count`). + */ + struct FffMixedItem *items; + /** + * Pointer to a heap-allocated array of `FffScore` (length = `count`). + */ + struct FffScore *scores; + /** + * Number of items/scores in the arrays. + */ + uint32_t count; + /** + * Total number of items (files + dirs) that matched the query. + */ + uint32_t total_matched; + /** + * Total number of indexed files. + */ + uint32_t total_files; + /** + * Total number of indexed directories. + */ + uint32_t total_dirs; + /** + * Location parsed from the query string. + */ + struct FffLocation location; +} FffMixedSearchResult; + +/** + * Create a new file finder instance (legacy 8-arg positional signature). + * + * @deprecated Use [`fff_create_instance_with`] (or + * [`fff_create_instance_with_value`] for FFI bindings) — both take the + * versioned [`FffCreateOptions`] struct that evolves without ABI breaks. + * This function delegates to `fff_create_instance_with` internally; the + * `use_unsafe_no_lock` parameter is deprecated and ignored. + * + * ## Safety + * See `fff_create_instance_with`. + */ +__attribute__((deprecated("Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."))) +struct FffResult *fff_create_instance(const char *base_path, + const char *frecency_db_path, + const char *history_db_path, + bool _use_unsafe_no_lock, + bool enable_mmap_cache, + bool enable_content_indexing, + bool watch, + bool ai_mode); + +/** + * Create a new file finder instance (legacy 13-arg positional signature). + * + * @deprecated Use [`fff_create_instance_with`] (or + * [`fff_create_instance_with_value`] for FFI bindings) — both take the + * versioned [`FffCreateOptions`] struct that evolves without ABI breaks. + * The `use_unsafe_no_lock` parameter is deprecated and ignored. + * + * ## Safety + * See `fff_create_instance_with`. + */ +__attribute__((deprecated("Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."))) +struct FffResult *fff_create_instance2(const char *base_path, + const char *frecency_db_path, + const char *history_db_path, + bool _use_unsafe_no_lock, + bool enable_mmap_cache, + bool enable_content_indexing, + bool watch, + bool ai_mode, + const char *log_file_path, + const char *log_level, + uint64_t cache_budget_max_files, + uint64_t cache_budget_max_bytes, + uint64_t cache_budget_max_file_size); + +/** + * Create a new file finder instance from an [`FffCreateOptions`] struct. + * + * **Direct C consumers** populate the struct (designated initializers + * recommended), set `version` to [`FFF_CREATE_OPTIONS_VERSION`], and pass + * it by pointer. New fields are appended in future versions; old callers + * passing `version = 1` keep working forever. + * + * **FFI consumers** that prefer struct-by-value semantics (e.g. ffi-rs's + * `paramsType: [structDef]`) should use [`fff_create_instance_with_value`] + * instead — it's a thin calling-convention adapter that delegates here. + * + * Required: `opts.base_path` must be non-NULL and non-empty. + * + * When all three `cache_budget_*` values are 0 the budget is auto-computed + * from repo size after the initial scan. Otherwise an explicit budget is + * used: any field left at 0 falls back to its `unlimited()` default. + * + * ## Safety + * * `opts` must be a valid pointer to an `FffCreateOptions` whose `version` + * is in the range `1..=FFF_CREATE_OPTIONS_VERSION`. + * * All string pointers inside `opts` must be valid null-terminated UTF-8 + * or NULL. + */ +struct FffResult *fff_create_instance_with(const struct FffCreateOptions *opts); + +/** + * Calling-convention adapter for [`fff_create_instance_with`]. + * + * Same logic, but takes the [`FffCreateOptions`] struct **by value**. This + * makes the function callable from FFI libraries whose native struct + * support passes structs by value on the wire (e.g. Node's `ffi-rs` with + * `paramsType: [structDef]`). + * + * This is **not** a versioned wrapper — when new fields are appended to + * `FffCreateOptions`, both this function and `fff_create_instance_with` + * pick them up automatically with no signature change. + * + * ## Safety + * All `*const c_char` fields inside `opts` must be valid null-terminated + * UTF-8 or NULL. The struct itself is consumed by value. + */ +struct FffResult *fff_create_instance_with_value(struct FffCreateOptions opts); + +/** + * Destroy a file finder instance and free all its resources. + * + * ## Safety + * `fff_handle` must be a valid pointer returned by `fff_create_instance`, or null (no-op). + */ +void fff_destroy(void *fff_handle); + +/** + * Perform fuzzy search on indexed files. + * + * # Parameters + * + * * `fff_handle` – instance from `fff_create_instance` + * * `query` – search query string + * * `current_file` – path of the currently open file for deprioritization (NULL/empty to skip) + * * `max_threads` – maximum worker threads (0 = auto-detect) + * * `page_index` – pagination offset (0 = first page) + * * `page_size` – results per page (0 = default 100) + * * `combo_boost_multiplier` – score multiplier for combo matches (0 = default 100) + * * `min_combo_count` – minimum combo count before boost applies (0 = default 3) + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. + */ +struct FffResult *fff_search(void *fff_handle, + const char *query, + const char *current_file, + uint32_t max_threads, + uint32_t page_index, + uint32_t page_size, + int32_t combo_boost_multiplier, + uint32_t min_combo_count); + +/** + * Glob-only search: filter indexed files by a single glob pattern, rank by + * frecency, and paginate. Bypasses the regular query parser entirely. + * + * Use this when you already have a literal glob pattern (e.g. `*.rs`, a + * recursive `**` match, or `src/components` prefix) and want neither fuzzy + * matching nor multi-token constraint parsing. Ranking falls back to + * frecency because there is no fuzzy score to combine with. + * + * # Parameters + * + * * `fff_handle` - instance from `fff_create_instance` + * * `pattern` - glob pattern (required, no parsing - passed through verbatim) + * * `current_file` - path of the currently open file for deprioritization (NULL/empty to skip) + * * `max_threads` - maximum worker threads (0 = auto-detect) + * * `page_index` - pagination offset (0 = first page) + * * `page_size` - results per page (0 = default 100) + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `pattern` and `current_file` must be valid null-terminated UTF-8 strings or NULL. + */ +struct FffResult *fff_glob(void *fff_handle, + const char *pattern, + const char *current_file, + uint32_t max_threads, + uint32_t page_index, + uint32_t page_size); + +/** + * Perform fuzzy search on indexed directories. + * + * # Parameters + * + * * `fff_handle` – instance from `fff_create_instance` + * * `query` – search query string + * * `current_file` – path of the currently open file for distance scoring (NULL/empty to skip) + * * `max_threads` – maximum worker threads (0 = auto-detect) + * * `page_index` – pagination offset (0 = first page) + * * `page_size` – results per page (0 = default 100) + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. + */ +struct FffResult *fff_search_directories(void *fff_handle, + const char *query, + const char *current_file, + uint32_t max_threads, + uint32_t page_index, + uint32_t page_size); + +/** + * Perform a mixed fuzzy search across both files and directories. + * + * Returns a single flat list where files and directories are interleaved + * by total score in descending order. Each item has an `item_type` field + * (0 = file, 1 = directory). + * + * # Parameters + * + * * `fff_handle` – instance from `fff_create_instance` + * * `query` – search query string + * * `current_file` – path of the currently open file (NULL/empty to skip) + * * `max_threads` – maximum worker threads (0 = auto-detect) + * * `page_index` – pagination offset (0 = first page) + * * `page_size` – results per page (0 = default 100) + * * `combo_boost_multiplier` – score multiplier for combo matches (0 = default 100) + * * `min_combo_count` – minimum combo count before boost applies (0 = default 3) + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. + */ +struct FffResult *fff_search_mixed(void *fff_handle, + const char *query, + const char *current_file, + uint32_t max_threads, + uint32_t page_index, + uint32_t page_size, + int32_t combo_boost_multiplier, + uint32_t min_combo_count); + +/** + * Perform content search (grep) across indexed files. + * + * # Parameters + * + * * `fff_handle` – instance from `fff_create_instance` + * * `query` – search query (supports constraint syntax like `*.rs pattern`) + * * `mode` – 0 = plain text (SIMD), 1 = regex, 2 = fuzzy + * * `max_file_size` – skip files larger than this in bytes (0 = default 10 MB) + * * `max_matches_per_file` – max matches per file (0 = unlimited) + * * `smart_case` – case-insensitive when query is all lowercase + * * `file_offset` – file-based pagination offset (0 = start) + * * `page_limit` – max matches to return (0 = default 50) + * * `time_budget_ms` – wall-clock budget in ms (0 = unlimited) + * * `before_context` – context lines before each match + * * `after_context` – context lines after each match + * * `classify_definitions` – tag matches that are code definitions + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `query` must be a valid null-terminated UTF-8 string. + */ +struct FffResult *fff_live_grep(void *fff_handle, + const char *query, + uint8_t mode, + uint64_t max_file_size, + uint32_t max_matches_per_file, + bool smart_case, + uint32_t file_offset, + uint32_t page_limit, + uint64_t time_budget_ms, + uint32_t before_context, + uint32_t after_context, + bool classify_definitions); + +/** + * Perform multi-pattern OR search (Aho-Corasick) across indexed files. + * + * Searches for lines matching ANY of the provided patterns using + * SIMD-accelerated multi-needle matching. + * + * # Parameters + * + * * `fff_handle` – instance from `fff_create_instance` + * * `patterns_joined` – patterns separated by `\n` (e.g. `"foo\nbar\nbaz"`) + * * `constraints` – file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip) + * * `max_file_size` – skip files larger than this in bytes (0 = default 10 MB) + * * `max_matches_per_file` – max matches per file (0 = unlimited) + * * `smart_case` – case-insensitive when all patterns are lowercase + * * `file_offset` – file-based pagination offset (0 = start) + * * `page_limit` – max matches to return (0 = default 50) + * * `time_budget_ms` – wall-clock budget in ms (0 = unlimited) + * * `before_context` – context lines before each match + * * `after_context` – context lines after each match + * * `classify_definitions` – tag matches that are code definitions + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `patterns_joined` and `constraints` must be valid null-terminated UTF-8 or NULL. + */ +struct FffResult *fff_multi_grep(void *fff_handle, + const char *patterns_joined, + const char *constraints, + uint64_t max_file_size, + uint32_t max_matches_per_file, + bool smart_case, + uint32_t file_offset, + uint32_t page_limit, + uint64_t time_budget_ms, + uint32_t before_context, + uint32_t after_context, + bool classify_definitions); + +/** + * Trigger a rescan of the file index. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_scan_files(void *fff_handle); + +/** + * Check if a scan is currently in progress. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +bool fff_is_scanning(void *fff_handle); + +/** + * Get the base path of the file picker. + * + * Returns an `FffResult` with a heap-allocated C string in the `handle` + * field. Free the string with `fff_free_string` after reading it. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_get_base_path(void *fff_handle); + +/** + * Get scan progress information. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_get_scan_progress(void *fff_handle); + +/** + * Wait for initial scan to complete. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_wait_for_scan(void *fff_handle, uint64_t timeout_ms); + +/** + * Wait for the background file watcher to be ready. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_wait_for_watcher(void *fff_handle, uint64_t timeout_ms); + +/** + * Restart indexing in a new directory. + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `new_path` must be a valid null-terminated UTF-8 string. + */ +struct FffResult *fff_restart_index(void *fff_handle, const char *new_path); + +/** + * Refresh git status cache. + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_refresh_git_status(void *fff_handle); + +/** + * Track query completion for smart suggestions. + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + * * `query` and `file_path` must be valid null-terminated UTF-8 strings. + */ +struct FffResult *fff_track_query(void *fff_handle, const char *query, const char *file_path); + +/** + * Get historical query by offset (0 = most recent). + * + * ## Safety + * `fff_handle` must be a valid instance pointer from `fff_create_instance`. + */ +struct FffResult *fff_get_historical_query(void *fff_handle, uint64_t offset); + +/** + * Get health check information. + * + * ## Safety + * * `fff_handle` must be a valid instance pointer from `fff_create_instance`, or null for + * a limited health check (version + git only). + * * `test_path` can be null or a valid null-terminated UTF-8 string. + */ +struct FffResult *fff_health_check(void *fff_handle, const char *test_path); + +/** + * Free a search result returned by `fff_search`. + * + * This frees the `FffSearchResult` struct, its `items` and `scores` arrays, + * and all heap-allocated strings within each item and score. + * + * ## Safety + * `result` must be a valid pointer previously returned via `FffResult.handle` + * from `fff_search`, or null (no-op). + */ +void fff_free_search_result(struct FffSearchResult *result); + +/** + * Get a pointer to the `index`-th `FffFileItem` in a search result. + * + * Returns null if `result` is null or `index >= result->count`. + * The returned pointer is valid until the search result is freed. + * + * ## Safety + * `result` must be a valid `FffSearchResult` pointer from `fff_search`. + */ +const struct FffFileItem *fff_search_result_get_item(const struct FffSearchResult *result, + uint32_t index); + +/** + * Get a pointer to the `index`-th `FffScore` in a search result. + * + * Returns null if `result` is null or `index >= result->count`. + * The returned pointer is valid until the search result is freed. + * + * ## Safety + * `result` must be a valid `FffSearchResult` pointer from `fff_search`. + */ +const struct FffScore *fff_search_result_get_score(const struct FffSearchResult *result, + uint32_t index); + +/** + * Free a grep result returned by `fff_live_grep` or `fff_multi_grep`. + * + * This frees the `FffGrepResult` struct, its `items` array, and all + * heap-allocated strings, match ranges, and context arrays within each match. + * + * ## Safety + * `result` must be a valid pointer previously returned via `FffResult.handle` + * from `fff_live_grep` or `fff_multi_grep`, or null (no-op). + */ +void fff_free_grep_result(struct FffGrepResult *result); + +/** + * Get a pointer to the `index`-th `FffGrepMatch` in a grep result. + * + * Returns null if `result` is null or `index >= result->count`. + * The returned pointer is valid until the grep result is freed. + * + * ## Safety + * `result` must be a valid `FffGrepResult` pointer from `fff_live_grep` or `fff_multi_grep`. + */ +const struct FffGrepMatch *fff_grep_result_get_match(const struct FffGrepResult *result, + uint32_t index); + +/** + * Free a scan progress result returned by `fff_get_scan_progress`. + * + * ## Safety + * `result` must be a valid pointer previously returned via `FffResult.handle` + * from `fff_get_scan_progress`, or null (no-op). + */ +void fff_free_scan_progress(struct FffScanProgress *result); + +/** + * Offset a pointer by `byte_offset` bytes. + * + * General-purpose utility for FFI consumers that need pointer arithmetic + * (e.g. iterating over arrays). Returns null if `base` is null. + * + * ## Safety + * The resulting pointer must be within the bounds of the original allocation. + */ +const void *fff_ptr_offset(const void *base, uintptr_t byte_offset); + +/** + * Free a result returned by any `fff_*` function. + * **IMPORTANT:** this doesn't clean the the internal handle, so it is safe to call right after + * you handle the error case. + * + * Note: Many non-libffi implementations are not supporting struct-by-value returns, so it's more + * convenient to have pointer returned at most of the time, though allocating result for every call + * is annoying, so we just rely on the fact that our allocator is good enough. + * + * ## Safety + * `result_ptr` must be a valid pointer returned by a `fff_*` function. + */ +void fff_free_result(struct FffResult *result_ptr); + +/** + * Free a string returned by `fff_*` functions. + * + * ## Safety + * `s` must be a valid C string allocated by this library. + */ +void fff_free_string(char *s); + +/** + * Free a directory search result returned by `fff_search_directories`. + * + * ## Safety + * `result` must be a valid pointer previously returned via `FffResult.handle` + * from `fff_search_directories`, or null (no-op). + */ +void fff_free_dir_search_result(struct FffDirSearchResult *result); + +/** + * Get a pointer to the `index`-th `FffDirItem` in a directory search result. + * + * ## Safety + * `result` must be a valid `FffDirSearchResult` pointer from `fff_search_directories`. + */ +const struct FffDirItem *fff_dir_search_result_get_item(const struct FffDirSearchResult *result, + uint32_t index); + +/** + * Get a pointer to the `index`-th `FffScore` in a directory search result. + * + * ## Safety + * `result` must be a valid `FffDirSearchResult` pointer from `fff_search_directories`. + */ +const struct FffScore *fff_dir_search_result_get_score(const struct FffDirSearchResult *result, + uint32_t index); + +/** + * Free a mixed search result returned by `fff_search_mixed`. + * + * ## Safety + * `result` must be a valid pointer previously returned via `FffResult.handle` + * from `fff_search_mixed`, or null (no-op). + */ +void fff_free_mixed_search_result(struct FffMixedSearchResult *result); + +/** + * Get a pointer to the `index`-th `FffMixedItem` in a mixed search result. + * + * ## Safety + * `result` must be a valid `FffMixedSearchResult` pointer from `fff_search_mixed`. + */ +const struct FffMixedItem *fff_mixed_search_result_get_item(const struct FffMixedSearchResult *result, + uint32_t index); + +/** + * Get a pointer to the `index`-th `FffScore` in a mixed search result. + * + * ## Safety + * `result` must be a valid `FffMixedSearchResult` pointer from `fff_search_mixed`. + */ +const struct FffScore *fff_mixed_search_result_get_score(const struct FffMixedSearchResult *result, + uint32_t index); + +/** + * Returns the relative path of a file item (e.g. `"src/main.rs"`). + * + * Returns null if `item` is null. The returned pointer is valid for the + * lifetime of the owning `FffSearchResult`; do not free it directly. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +const char *fff_file_item_get_relative_path(const struct FffFileItem *item); + +/** + * Returns the file-name component of a file item (e.g. `"main.rs"`). + * + * Returns null if `item` is null. Do not free the returned pointer. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +const char *fff_file_item_get_file_name(const struct FffFileItem *item); + +/** + * Returns the git status string for a file item (e.g. `"M "`, `"??"`) + * or null if git is unavailable, the file is untracked, or `item` is null. + * + * Do not free the returned pointer. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +const char *fff_file_item_get_git_status(const struct FffFileItem *item); + +/** + * Returns the file size in bytes. Returns `0` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +uint64_t fff_file_item_get_size(const struct FffFileItem *item); + +/** + * Returns the last-modified time as seconds since the UNIX epoch. + * Returns `0` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +uint64_t fff_file_item_get_modified(const struct FffFileItem *item); + +/** + * Returns the combined frecency score. Returns `0` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +int64_t fff_file_item_get_total_frecency_score(const struct FffFileItem *item); + +/** + * Returns the access-based frecency score. Returns `0` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +int64_t fff_file_item_get_access_frecency_score(const struct FffFileItem *item); + +/** + * Returns the modification-based frecency score. Returns `0` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +int64_t fff_file_item_get_modification_frecency_score(const struct FffFileItem *item); + +/** + * Returns `true` if the file was detected as binary. Returns `false` if `item` is null. + * + * ## Safety + * `item` must be a valid `FffFileItem` pointer or null. + */ +bool fff_file_item_get_is_binary(const struct FffFileItem *item); + +/** + * Returns the relative path of the file containing this grep match. + * + * Returns null if `m` is null. Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_relative_path(const struct FffGrepMatch *m); + +/** + * Returns the file-name component of the file containing this grep match. + * + * Returns null if `m` is null. Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_file_name(const struct FffGrepMatch *m); + +/** + * Returns the git status string for the matched file (e.g. `"M "`, `"??"`) + * or null if git is unavailable, the file is untracked, or `m` is null. + * + * Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_git_status(const struct FffGrepMatch *m); + +/** + * Returns the full text content of the matched line. + * + * Returns null if `m` is null. Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_line_content(const struct FffGrepMatch *m); + +/** + * Returns the 1-based line number of the match within its file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint64_t fff_grep_match_get_line_number(const struct FffGrepMatch *m); + +/** + * Returns the 0-based column of the match start within its line. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint32_t fff_grep_match_get_col(const struct FffGrepMatch *m); + +/** + * Returns the byte offset of the match start from the beginning of the file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint64_t fff_grep_match_get_byte_offset(const struct FffGrepMatch *m); + +/** + * Returns the file size in bytes for the matched file. Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint64_t fff_grep_match_get_size(const struct FffGrepMatch *m); + +/** + * Returns the combined frecency score for the matched file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +int64_t fff_grep_match_get_total_frecency_score(const struct FffGrepMatch *m); + +/** + * Returns the access-based frecency score for the matched file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +int64_t fff_grep_match_get_access_frecency_score(const struct FffGrepMatch *m); + +/** + * Returns the modification-based frecency score for the matched file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +int64_t fff_grep_match_get_modification_frecency_score(const struct FffGrepMatch *m); + +/** + * Returns the last-modified time as seconds since the UNIX epoch for the matched file. + * Returns `0` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint64_t fff_grep_match_get_modified(const struct FffGrepMatch *m); + +/** + * Returns the number of highlight ranges in this match. Returns `0` if `m` is null. + * + * Use with [`fff_grep_match_get_match_range`] to iterate the highlight spans. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint32_t fff_grep_match_get_match_ranges_count(const struct FffGrepMatch *m); + +/** + * Returns a pointer to the `index`-th [`FffMatchRange`] highlight span. + * + * Returns null if `m` is null, `index >= match_ranges_count`, or the + * ranges array is null. The returned pointer is valid until the owning + * `FffGrepResult` is freed; do not free it directly. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const struct FffMatchRange *fff_grep_match_get_match_range(const struct FffGrepMatch *m, + uint32_t index); + +/** + * Returns the number of context lines captured before the match. + * Returns `0` if `m` is null. + * + * Use with [`fff_grep_match_get_context_before`] to read each line. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint32_t fff_grep_match_get_context_before_count(const struct FffGrepMatch *m); + +/** + * Returns the `index`-th context line before the match. + * + * Returns null if `m` is null, `index >= context_before_count`, or the + * context array is null. Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_context_before(const struct FffGrepMatch *m, uint32_t index); + +/** + * Returns the number of context lines captured after the match. + * Returns `0` if `m` is null. + * + * Use with [`fff_grep_match_get_context_after`] to read each line. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint32_t fff_grep_match_get_context_after_count(const struct FffGrepMatch *m); + +/** + * Returns the `index`-th context line after the match. + * + * Returns null if `m` is null, `index >= context_after_count`, or the + * context array is null. Do not free the returned pointer. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +const char *fff_grep_match_get_context_after(const struct FffGrepMatch *m, uint32_t index); + +/** + * Returns the fuzzy match score. Returns `0` if `m` is null or no fuzzy + * score is present. + * + * Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is + * ambiguous without that flag. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +uint16_t fff_grep_match_get_fuzzy_score(const struct FffGrepMatch *m); + +/** + * Returns `true` if this match carries a valid fuzzy score. + * Returns `false` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +bool fff_grep_match_get_has_fuzzy_score(const struct FffGrepMatch *m); + +/** + * Returns `true` if the match was identified as a symbol definition. + * Returns `false` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +bool fff_grep_match_get_is_definition(const struct FffGrepMatch *m); + +/** + * Returns `true` if the matched file was detected as binary. + * Returns `false` if `m` is null. + * + * ## Safety + * `m` must be a valid `FffGrepMatch` pointer or null. + */ +bool fff_grep_match_get_is_binary(const struct FffGrepMatch *m); + +/** + * Returns the number of items in the result. Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffSearchResult` pointer or null. + */ +uint32_t fff_search_result_get_count(const struct FffSearchResult *r); + +/** + * Returns the total number of files that matched before the result was + * truncated to the page size. Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffSearchResult` pointer or null. + */ +uint32_t fff_search_result_get_total_matched(const struct FffSearchResult *r); + +/** + * Returns the total number of indexed files considered during search. + * Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffSearchResult` pointer or null. + */ +uint32_t fff_search_result_get_total_files(const struct FffSearchResult *r); + +/** + * Returns the number of matches in the result. Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_count(const struct FffGrepResult *r); + +/** + * Returns the total number of matches found across all pages. + * Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_total_matched(const struct FffGrepResult *r); + +/** + * Returns the number of files actually opened and searched in this call. + * Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_total_files_searched(const struct FffGrepResult *r); + +/** + * Returns the total number of indexed files before any filtering. + * Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_total_files(const struct FffGrepResult *r); + +/** + * Returns the number of files eligible for search after path/type filtering. + * Returns `0` if `r` is null. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_filtered_file_count(const struct FffGrepResult *r); + +/** + * Returns the file offset for the next page, or `0` if all files have been + * searched or `r` is null. Pass this value as `file_offset` to a subsequent + * `fff_live_grep` or `fff_multi_grep` call to continue pagination. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +uint32_t fff_grep_result_get_next_file_offset(const struct FffGrepResult *r); + +/** + * Returns the regex compilation error string if the engine fell back to + * literal matching, or null if there was no error or `r` is null. + * + * Do not free the returned pointer. + * + * ## Safety + * `r` must be a valid `FffGrepResult` pointer or null. + */ +const char *fff_grep_result_get_regex_fallback_error(const struct FffGrepResult *r); + +#endif /* FFF_C_H */ diff --git a/crates/fff-c/src/accessors.rs b/crates/fff-c/src/accessors.rs new file mode 100644 index 0000000..e7e5a59 --- /dev/null +++ b/crates/fff-c/src/accessors.rs @@ -0,0 +1,882 @@ +//! Stable accessor functions for `fff-c` FFI struct fields. +//! +//! # Why this exists +//! +//! `fff-c` exposes its result types as plain `#[repr(C)]` structs. External +//! consumers (Emacs Lisp via `emacs-ffi`, Python `ctypes`, etc.) that access +//! fields by hardcoding byte offsets break silently whenever the struct layout +//! changes — a new field shifts every subsequent offset with no compile-time +//! warning. +//! +//! These functions turn field access into a **stable named API**: callers bind +//! to a symbol name once and are fully insulated from layout changes. +//! +//! # Usage from Emacs Lisp (example) +//! +//! ```elisp +//! (define-ffi-function fff--grep-match-line-content +//! "fff_grep_match_get_line_content" :pointer [:pointer] fff--library) +//! +//! (ffi-get-c-string (fff--grep-match-line-content match-ptr)) +//! ``` +//! +//! # Array iteration +//! +//! To walk result arrays use `fff_search_result_get_item`, +//! `fff_grep_result_get_match`, and `fff_search_result_get_score` — these are +//! defined in the main `lib.rs` FFI surface alongside the search functions. + +use std::ffi::c_char; +use std::ptr; + +use crate::ffi_types::{FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffSearchResult}; + +// ── FffFileItem ────────────────────────────────────────────────────────────── + +/// Returns the relative path of a file item (e.g. `"src/main.rs"`). +/// +/// Returns null if `item` is null. The returned pointer is valid for the +/// lifetime of the owning `FffSearchResult`; do not free it directly. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_relative_path( + item: *const FffFileItem, +) -> *const c_char { + if item.is_null() { + return ptr::null(); + } + unsafe { (*item).relative_path } +} + +/// Returns the file-name component of a file item (e.g. `"main.rs"`). +/// +/// Returns null if `item` is null. Do not free the returned pointer. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_file_name(item: *const FffFileItem) -> *const c_char { + if item.is_null() { + return ptr::null(); + } + unsafe { (*item).file_name } +} + +/// Returns the git status string for a file item (e.g. `"M "`, `"??"`) +/// or null if git is unavailable, the file is untracked, or `item` is null. +/// +/// Do not free the returned pointer. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_git_status(item: *const FffFileItem) -> *const c_char { + if item.is_null() { + return ptr::null(); + } + unsafe { (*item).git_status } +} + +/// Returns the file size in bytes. Returns `0` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_size(item: *const FffFileItem) -> u64 { + if item.is_null() { + return 0; + } + unsafe { (*item).size } +} + +/// Returns the last-modified time as seconds since the UNIX epoch. +/// Returns `0` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_modified(item: *const FffFileItem) -> u64 { + if item.is_null() { + return 0; + } + unsafe { (*item).modified } +} + +/// Returns the combined frecency score. Returns `0` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_total_frecency_score(item: *const FffFileItem) -> i64 { + if item.is_null() { + return 0; + } + unsafe { (*item).total_frecency_score } +} + +/// Returns the access-based frecency score. Returns `0` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_access_frecency_score(item: *const FffFileItem) -> i64 { + if item.is_null() { + return 0; + } + unsafe { (*item).access_frecency_score } +} + +/// Returns the modification-based frecency score. Returns `0` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_modification_frecency_score( + item: *const FffFileItem, +) -> i64 { + if item.is_null() { + return 0; + } + unsafe { (*item).modification_frecency_score } +} + +/// Returns `true` if the file was detected as binary. Returns `false` if `item` is null. +/// +/// ## Safety +/// `item` must be a valid `FffFileItem` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_file_item_get_is_binary(item: *const FffFileItem) -> bool { + if item.is_null() { + return false; + } + unsafe { (*item).is_binary } +} + +// ── FffGrepMatch ───────────────────────────────────────────────────────────── + +/// Returns the relative path of the file containing this grep match. +/// +/// Returns null if `m` is null. Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_relative_path(m: *const FffGrepMatch) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + unsafe { (*m).relative_path } +} + +/// Returns the file-name component of the file containing this grep match. +/// +/// Returns null if `m` is null. Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_file_name(m: *const FffGrepMatch) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + unsafe { (*m).file_name } +} + +/// Returns the git status string for the matched file (e.g. `"M "`, `"??"`) +/// or null if git is unavailable, the file is untracked, or `m` is null. +/// +/// Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_git_status(m: *const FffGrepMatch) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + unsafe { (*m).git_status } +} + +/// Returns the full text content of the matched line. +/// +/// Returns null if `m` is null. Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_line_content(m: *const FffGrepMatch) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + unsafe { (*m).line_content } +} + +/// Returns the 1-based line number of the match within its file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_line_number(m: *const FffGrepMatch) -> u64 { + if m.is_null() { + return 0; + } + unsafe { (*m).line_number } +} + +/// Returns the 0-based column of the match start within its line. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_col(m: *const FffGrepMatch) -> u32 { + if m.is_null() { + return 0; + } + unsafe { (*m).col } +} + +/// Returns the byte offset of the match start from the beginning of the file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_byte_offset(m: *const FffGrepMatch) -> u64 { + if m.is_null() { + return 0; + } + unsafe { (*m).byte_offset } +} + +/// Returns the file size in bytes for the matched file. Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_size(m: *const FffGrepMatch) -> u64 { + if m.is_null() { + return 0; + } + unsafe { (*m).size } +} + +/// Returns the combined frecency score for the matched file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_total_frecency_score(m: *const FffGrepMatch) -> i64 { + if m.is_null() { + return 0; + } + unsafe { (*m).total_frecency_score } +} + +/// Returns the access-based frecency score for the matched file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_access_frecency_score(m: *const FffGrepMatch) -> i64 { + if m.is_null() { + return 0; + } + unsafe { (*m).access_frecency_score } +} + +/// Returns the modification-based frecency score for the matched file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_modification_frecency_score( + m: *const FffGrepMatch, +) -> i64 { + if m.is_null() { + return 0; + } + unsafe { (*m).modification_frecency_score } +} + +/// Returns the last-modified time as seconds since the UNIX epoch for the matched file. +/// Returns `0` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_modified(m: *const FffGrepMatch) -> u64 { + if m.is_null() { + return 0; + } + unsafe { (*m).modified } +} + +/// Returns the number of highlight ranges in this match. Returns `0` if `m` is null. +/// +/// Use with [`fff_grep_match_get_match_range`] to iterate the highlight spans. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_match_ranges_count(m: *const FffGrepMatch) -> u32 { + if m.is_null() { + return 0; + } + unsafe { (*m).match_ranges_count } +} + +/// Returns a pointer to the `index`-th [`FffMatchRange`] highlight span. +/// +/// Returns null if `m` is null, `index >= match_ranges_count`, or the +/// ranges array is null. The returned pointer is valid until the owning +/// `FffGrepResult` is freed; do not free it directly. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_match_range( + m: *const FffGrepMatch, + index: u32, +) -> *const FffMatchRange { + if m.is_null() { + return ptr::null(); + } + let m = unsafe { &*m }; + if index >= m.match_ranges_count || m.match_ranges.is_null() { + return ptr::null(); + } + unsafe { m.match_ranges.add(index as usize) } +} + +/// Returns the number of context lines captured before the match. +/// Returns `0` if `m` is null. +/// +/// Use with [`fff_grep_match_get_context_before`] to read each line. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_context_before_count(m: *const FffGrepMatch) -> u32 { + if m.is_null() { + return 0; + } + unsafe { (*m).context_before_count } +} + +/// Returns the `index`-th context line before the match. +/// +/// Returns null if `m` is null, `index >= context_before_count`, or the +/// context array is null. Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_context_before( + m: *const FffGrepMatch, + index: u32, +) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + let m = unsafe { &*m }; + if index >= m.context_before_count || m.context_before.is_null() { + return ptr::null(); + } + unsafe { *m.context_before.add(index as usize) } +} + +/// Returns the number of context lines captured after the match. +/// Returns `0` if `m` is null. +/// +/// Use with [`fff_grep_match_get_context_after`] to read each line. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_context_after_count(m: *const FffGrepMatch) -> u32 { + if m.is_null() { + return 0; + } + unsafe { (*m).context_after_count } +} + +/// Returns the `index`-th context line after the match. +/// +/// Returns null if `m` is null, `index >= context_after_count`, or the +/// context array is null. Do not free the returned pointer. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_context_after( + m: *const FffGrepMatch, + index: u32, +) -> *const c_char { + if m.is_null() { + return ptr::null(); + } + let m = unsafe { &*m }; + if index >= m.context_after_count || m.context_after.is_null() { + return ptr::null(); + } + unsafe { *m.context_after.add(index as usize) } +} + +/// Returns the fuzzy match score. Returns `0` if `m` is null or no fuzzy +/// score is present. +/// +/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is +/// ambiguous without that flag. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_fuzzy_score(m: *const FffGrepMatch) -> u16 { + if m.is_null() { + return 0; + } + unsafe { (*m).fuzzy_score } +} + +/// Returns `true` if this match carries a valid fuzzy score. +/// Returns `false` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_has_fuzzy_score(m: *const FffGrepMatch) -> bool { + if m.is_null() { + return false; + } + unsafe { (*m).has_fuzzy_score } +} + +/// Returns `true` if the match was identified as a symbol definition. +/// Returns `false` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_is_definition(m: *const FffGrepMatch) -> bool { + if m.is_null() { + return false; + } + unsafe { (*m).is_definition } +} + +/// Returns `true` if the matched file was detected as binary. +/// Returns `false` if `m` is null. +/// +/// ## Safety +/// `m` must be a valid `FffGrepMatch` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_match_get_is_binary(m: *const FffGrepMatch) -> bool { + if m.is_null() { + return false; + } + unsafe { (*m).is_binary } +} + +// ── FffSearchResult ────────────────────────────────────────────────────────── + +/// Returns the number of items in the result. Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffSearchResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_result_get_count(r: *const FffSearchResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).count } +} + +/// Returns the total number of files that matched before the result was +/// truncated to the page size. Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffSearchResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_result_get_total_matched(r: *const FffSearchResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).total_matched } +} + +/// Returns the total number of indexed files considered during search. +/// Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffSearchResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_result_get_total_files(r: *const FffSearchResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).total_files } +} + +// ── FffGrepResult ───────────────────────────────────────────────────────────── + +/// Returns the number of matches in the result. Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_count(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).count } +} + +/// Returns the total number of matches found across all pages. +/// Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_total_matched(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).total_matched } +} + +/// Returns the number of files actually opened and searched in this call. +/// Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_total_files_searched(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).total_files_searched } +} + +/// Returns the total number of indexed files before any filtering. +/// Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_total_files(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).total_files } +} + +/// Returns the number of files eligible for search after path/type filtering. +/// Returns `0` if `r` is null. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_filtered_file_count(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).filtered_file_count } +} + +/// Returns the file offset for the next page, or `0` if all files have been +/// searched or `r` is null. Pass this value as `file_offset` to a subsequent +/// `fff_live_grep` or `fff_multi_grep` call to continue pagination. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_next_file_offset(r: *const FffGrepResult) -> u32 { + if r.is_null() { + return 0; + } + unsafe { (*r).next_file_offset } +} + +/// Returns the regex compilation error string if the engine fell back to +/// literal matching, or null if there was no error or `r` is null. +/// +/// Do not free the returned pointer. +/// +/// ## Safety +/// `r` must be a valid `FffGrepResult` pointer or null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_regex_fallback_error( + r: *const FffGrepResult, +) -> *const c_char { + if r.is_null() { + return ptr::null(); + } + unsafe { (*r).regex_fallback_error } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::ptr; + + // ── helpers ────────────────────────────────────────────────────────────── + + fn make_file_item(path: &str, name: &str) -> FffFileItem { + FffFileItem { + relative_path: CString::new(path).unwrap().into_raw(), + file_name: CString::new(name).unwrap().into_raw(), + git_status: ptr::null_mut(), + size: 1024, + modified: 1_700_000_000, + access_frecency_score: 10, + modification_frecency_score: 20, + total_frecency_score: 30, + is_binary: false, + } + } + + unsafe fn free_file_item(item: &mut FffFileItem) { + unsafe { + if !item.relative_path.is_null() { + drop(CString::from_raw(item.relative_path)); + } + if !item.file_name.is_null() { + drop(CString::from_raw(item.file_name)); + } + if !item.git_status.is_null() { + drop(CString::from_raw(item.git_status)); + } + } + } + + fn make_grep_match(path: &str, line: &str) -> FffGrepMatch { + FffGrepMatch { + relative_path: CString::new(path).unwrap().into_raw(), + file_name: CString::new("file.rs").unwrap().into_raw(), + git_status: ptr::null_mut(), + line_content: CString::new(line).unwrap().into_raw(), + match_ranges: ptr::null_mut(), + context_before: ptr::null_mut(), + context_after: ptr::null_mut(), + size: 512, + modified: 1_600_000_000, + total_frecency_score: 5, + access_frecency_score: 6, + modification_frecency_score: 7, + line_number: 42, + byte_offset: 100, + col: 8, + match_ranges_count: 0, + context_before_count: 0, + context_after_count: 0, + fuzzy_score: 0, + has_fuzzy_score: false, + is_binary: false, + is_definition: true, + } + } + + unsafe fn free_grep_match(m: &mut FffGrepMatch) { + unsafe { + if !m.relative_path.is_null() { + drop(CString::from_raw(m.relative_path)); + } + if !m.file_name.is_null() { + drop(CString::from_raw(m.file_name)); + } + if !m.line_content.is_null() { + drop(CString::from_raw(m.line_content)); + } + } + } + + fn make_search_result(count: u32, total: u32, files: u32) -> FffSearchResult { + FffSearchResult { + items: ptr::null_mut(), + scores: ptr::null_mut(), + count, + total_matched: total, + total_files: files, + location: crate::ffi_types::FffLocation { + tag: 0, + line: 0, + col: 0, + end_line: 0, + end_col: 0, + }, + } + } + + fn make_grep_result() -> FffGrepResult { + FffGrepResult { + items: ptr::null_mut(), + count: 3, + total_matched: 10, + total_files_searched: 50, + total_files: 200, + filtered_file_count: 80, + next_file_offset: 51, + regex_fallback_error: ptr::null_mut(), + } + } + + // ── null-guard tests: every function returns its zero-value on NULL ─────── + + #[test] + fn null_file_item_returns_null_or_zero() { + let null: *const FffFileItem = ptr::null(); + unsafe { + assert!(fff_file_item_get_relative_path(null).is_null()); + assert!(fff_file_item_get_file_name(null).is_null()); + assert!(fff_file_item_get_git_status(null).is_null()); + assert_eq!(fff_file_item_get_size(null), 0); + assert_eq!(fff_file_item_get_modified(null), 0); + assert_eq!(fff_file_item_get_access_frecency_score(null), 0); + assert_eq!(fff_file_item_get_modification_frecency_score(null), 0); + assert_eq!(fff_file_item_get_total_frecency_score(null), 0); + assert!(!fff_file_item_get_is_binary(null)); + } + } + + #[test] + fn null_grep_match_returns_null_or_zero() { + let null: *const FffGrepMatch = ptr::null(); + unsafe { + assert!(fff_grep_match_get_relative_path(null).is_null()); + assert!(fff_grep_match_get_file_name(null).is_null()); + assert!(fff_grep_match_get_git_status(null).is_null()); + assert!(fff_grep_match_get_line_content(null).is_null()); + assert_eq!(fff_grep_match_get_line_number(null), 0); + assert_eq!(fff_grep_match_get_byte_offset(null), 0); + assert_eq!(fff_grep_match_get_col(null), 0); + assert_eq!(fff_grep_match_get_size(null), 0); + assert_eq!(fff_grep_match_get_modified(null), 0); + assert_eq!(fff_grep_match_get_total_frecency_score(null), 0); + assert_eq!(fff_grep_match_get_access_frecency_score(null), 0); + assert_eq!(fff_grep_match_get_modification_frecency_score(null), 0); + assert_eq!(fff_grep_match_get_match_ranges_count(null), 0); + assert_eq!(fff_grep_match_get_context_before_count(null), 0); + assert_eq!(fff_grep_match_get_context_after_count(null), 0); + assert!(!fff_grep_match_get_has_fuzzy_score(null)); + assert_eq!(fff_grep_match_get_fuzzy_score(null), 0); + assert!(!fff_grep_match_get_is_binary(null)); + assert!(!fff_grep_match_get_is_definition(null)); + assert!(fff_grep_match_get_context_before(null, 0).is_null()); + assert!(fff_grep_match_get_context_after(null, 0).is_null()); + assert!(fff_grep_match_get_match_range(null, 0).is_null()); + } + } + + #[test] + fn null_search_result_returns_zero() { + let null: *const FffSearchResult = ptr::null(); + unsafe { + assert_eq!(fff_search_result_get_count(null), 0); + assert_eq!(fff_search_result_get_total_matched(null), 0); + assert_eq!(fff_search_result_get_total_files(null), 0); + } + } + + #[test] + fn null_grep_result_returns_zero_or_null() { + let null: *const FffGrepResult = ptr::null(); + unsafe { + assert_eq!(fff_grep_result_get_count(null), 0); + assert_eq!(fff_grep_result_get_total_matched(null), 0); + assert_eq!(fff_grep_result_get_total_files_searched(null), 0); + assert_eq!(fff_grep_result_get_total_files(null), 0); + assert_eq!(fff_grep_result_get_filtered_file_count(null), 0); + assert_eq!(fff_grep_result_get_next_file_offset(null), 0); + assert!(fff_grep_result_get_regex_fallback_error(null).is_null()); + } + } + + // ── data correctness tests ──────────────────────────────────────────────── + + #[test] + fn file_item_getters_return_correct_values() { + let mut item = make_file_item("src/main.rs", "main.rs"); + let p = &item as *const FffFileItem; + unsafe { + let path = std::ffi::CStr::from_ptr(fff_file_item_get_relative_path(p)); + assert_eq!(path.to_str().unwrap(), "src/main.rs"); + + let name = std::ffi::CStr::from_ptr(fff_file_item_get_file_name(p)); + assert_eq!(name.to_str().unwrap(), "main.rs"); + + assert!(fff_file_item_get_git_status(p).is_null()); + assert_eq!(fff_file_item_get_size(p), 1024); + assert_eq!(fff_file_item_get_modified(p), 1_700_000_000); + assert_eq!(fff_file_item_get_access_frecency_score(p), 10); + assert_eq!(fff_file_item_get_modification_frecency_score(p), 20); + assert_eq!(fff_file_item_get_total_frecency_score(p), 30); + assert!(!fff_file_item_get_is_binary(p)); + + free_file_item(&mut item); + } + } + + #[test] + fn grep_match_getters_return_correct_values() { + let mut m = make_grep_match("src/lib.rs", "fn hello()"); + let p = &m as *const FffGrepMatch; + unsafe { + let path = std::ffi::CStr::from_ptr(fff_grep_match_get_relative_path(p)); + assert_eq!(path.to_str().unwrap(), "src/lib.rs"); + + let line = std::ffi::CStr::from_ptr(fff_grep_match_get_line_content(p)); + assert_eq!(line.to_str().unwrap(), "fn hello()"); + + assert_eq!(fff_grep_match_get_line_number(p), 42); + assert_eq!(fff_grep_match_get_byte_offset(p), 100); + assert_eq!(fff_grep_match_get_col(p), 8); + assert_eq!(fff_grep_match_get_size(p), 512); + assert_eq!(fff_grep_match_get_modified(p), 1_600_000_000); + assert_eq!(fff_grep_match_get_total_frecency_score(p), 5); + assert_eq!(fff_grep_match_get_access_frecency_score(p), 6); + assert_eq!(fff_grep_match_get_modification_frecency_score(p), 7); + assert_eq!(fff_grep_match_get_match_ranges_count(p), 0); + assert!(!fff_grep_match_get_has_fuzzy_score(p)); + assert!(!fff_grep_match_get_is_binary(p)); + assert!(fff_grep_match_get_is_definition(p)); + + free_grep_match(&mut m); + } + } + + #[test] + fn search_result_getters_return_correct_values() { + let r = make_search_result(5, 20, 100); + let p = &r as *const FffSearchResult; + unsafe { + assert_eq!(fff_search_result_get_count(p), 5); + assert_eq!(fff_search_result_get_total_matched(p), 20); + assert_eq!(fff_search_result_get_total_files(p), 100); + } + } + + #[test] + fn grep_result_getters_return_correct_values() { + let r = make_grep_result(); + let p = &r as *const FffGrepResult; + unsafe { + assert_eq!(fff_grep_result_get_count(p), 3); + assert_eq!(fff_grep_result_get_total_matched(p), 10); + assert_eq!(fff_grep_result_get_total_files_searched(p), 50); + assert_eq!(fff_grep_result_get_total_files(p), 200); + assert_eq!(fff_grep_result_get_filtered_file_count(p), 80); + assert_eq!(fff_grep_result_get_next_file_offset(p), 51); + assert!(fff_grep_result_get_regex_fallback_error(p).is_null()); + } + } +} diff --git a/crates/fff-c/src/ffi_types.rs b/crates/fff-c/src/ffi_types.rs new file mode 100644 index 0000000..6421de7 --- /dev/null +++ b/crates/fff-c/src/ffi_types.rs @@ -0,0 +1,831 @@ +//! FFI-compatible type definitions +//! +//! All result types use `#[repr(C)]` structs for direct memory access from any +//! language with C FFI support. No JSON serialization is used for search or grep +//! results — callers read struct fields directly. + +use std::ffi::{CString, c_char, c_void}; +use std::ptr; + +use fff::file_picker::FilePicker; +use fff::git::format_git_status; +use fff::{ + DirItem, DirSearchResult, FileItem, GrepMatch, GrepResult, Location, MixedItemRef, + MixedSearchResult, Score, SearchResult, +}; + +/// Current used version of [`FffCreateOptions`]. +pub const FFF_CREATE_OPTIONS_VERSION: u32 = 2; + +/// Options for `fff_create_instance_with`. +/// +/// Versioned struct: you populate the struct at your call level, we guarantee that +/// the version is stable across the version changes, new fields only appended! +#[repr(C)] +pub struct FffCreateOptions { + /// Set to [`FFF_CREATE_OPTIONS_VERSION`] when allocating. Used by the + /// library to determine which trailing fields are populated. + pub version: u32, + /// Directory to index (required, non-NULL). + pub base_path: *const c_char, + /// Frecency LMDB database path. NULL/empty to skip frecency tracking. + pub frecency_db_path: *const c_char, + /// Query history LMDB database path. NULL/empty to skip query tracking. + pub history_db_path: *const c_char, + /// Pre-populate mmap caches for top-frecency files after the initial scan. + pub enable_mmap_cache: bool, + /// Build content index after the initial scan for faster grep. + pub enable_content_indexing: bool, + /// Start a background file-system watcher for live updates. + pub watch: bool, + /// Enable AI-agent optimizations. + pub ai_mode: bool, + /// Tracing log file path. NULL/empty to skip log init. + pub log_file_path: *const c_char, + /// Log level: `"trace" | "debug" | "info" | "warn" | "error"`. + /// NULL/empty defaults to `"info"`. Ignored when `log_file_path` is unset. + pub log_level: *const c_char, + /// Content cache file-count cap. 0 = auto. + pub cache_budget_max_files: u64, + /// Content cache byte cap. 0 = auto. + pub cache_budget_max_bytes: u64, + /// Per-file byte cap inside the content cache. 0 = auto. + pub cache_budget_max_file_size: u64, + /// Allow indexing the filesystem root (`/`). Off by default — root is + /// rarely the intended target and floods the watcher with churn. + pub enable_fs_root_scanning: bool, + /// Allow indexing the user's home directory. Same trade-off as + /// `enable_fs_root_scanning`. + pub enable_home_dir_scanning: bool, + // ----- v2 fields ----- + /// Follow symlinks during scan and watcher walks. Off by default — + /// enabling this without external loop protection can wedge the watcher + /// on cyclic symlink graphs. Caller is responsible for the trade-off. + pub follow_symlinks: bool, + // ----- new version 3+ fields go here, ALWAYS appended ----- +} + +impl FffCreateOptions { + /// Default values for a v1 options struct. + pub fn defaults() -> Self { + Self { + version: FFF_CREATE_OPTIONS_VERSION, + base_path: ptr::null(), + frecency_db_path: ptr::null(), + history_db_path: ptr::null(), + enable_mmap_cache: true, + enable_content_indexing: true, + watch: true, + ai_mode: false, + log_file_path: ptr::null(), + log_level: ptr::null(), + cache_budget_max_files: 0, + cache_budget_max_bytes: 0, + cache_budget_max_file_size: 0, + enable_fs_root_scanning: false, + enable_home_dir_scanning: false, + follow_symlinks: false, + } + } +} + +/// Allocate a heap CString from a `&str`, returning a raw pointer. +fn cstring_new(s: &str) -> *mut c_char { + CString::new(s).unwrap_or_default().into_raw() +} + +/// Convert a `Vec` into a raw pointer + count, leaking the memory. +fn vec_to_raw(v: Vec) -> (*mut T, u32) { + if v.is_empty() { + return (ptr::null_mut(), 0); + } + let count = v.len() as u32; + let mut boxed = v.into_boxed_slice(); + let p = boxed.as_mut_ptr(); + std::mem::forget(boxed); + (p, count) +} + +/// Convert a `&[String]` into a heap-allocated array of C strings. +fn strings_to_raw(v: &[String]) -> (*mut *mut c_char, u32) { + if v.is_empty() { + return (ptr::null_mut(), 0); + } + let ptrs: Vec<*mut c_char> = v.iter().map(|s| cstring_new(s)).collect(); + vec_to_raw(ptrs) +} + +/// Free a heap-allocated array of C strings. +/// +/// ## Safety +/// `arr` must have been produced by `strings_to_raw`. +unsafe fn free_cstring_array(arr: *mut *mut c_char, count: u32) { + if arr.is_null() { + return; + } + unsafe { + let ptrs = Vec::from_raw_parts(arr, count as usize, count as usize); + for p in ptrs { + if !p.is_null() { + drop(CString::from_raw(p)); + } + } + } +} + +/// A file item returned by `fff_search`. +/// +/// All string fields are heap-allocated and owned by the parent `FffSearchResult`. +/// Free the entire result with `fff_free_search_result`. +#[repr(C)] +pub struct FffFileItem { + pub relative_path: *mut c_char, + pub file_name: *mut c_char, + pub git_status: *mut c_char, + pub size: u64, + pub modified: u64, + pub access_frecency_score: i64, + pub modification_frecency_score: i64, + pub total_frecency_score: i64, + pub is_binary: bool, +} + +impl FffFileItem { + pub fn from_item(item: &FileItem, picker: &FilePicker) -> Self { + FffFileItem { + relative_path: cstring_new(&item.relative_path(picker)), + file_name: cstring_new(&item.file_name(picker)), + git_status: cstring_new(format_git_status(item.git_status)), + size: item.size, + modified: item.modified, + access_frecency_score: item.access_frecency_score as i64, + modification_frecency_score: item.modification_frecency_score as i64, + total_frecency_score: item.total_frecency_score() as i64, + is_binary: item.is_binary(), + } + } +} + +impl FffFileItem { + /// ## Safety + /// All string pointers must have been allocated by `CString::into_raw`. + pub unsafe fn free_strings(&mut self) { + unsafe { + if !self.relative_path.is_null() { + drop(CString::from_raw(self.relative_path)); + } + if !self.file_name.is_null() { + drop(CString::from_raw(self.file_name)); + } + if !self.git_status.is_null() { + drop(CString::from_raw(self.git_status)); + } + } + } +} + +/// Score breakdown for a search result. +#[repr(C)] +pub struct FffScore { + pub total: i32, + pub base_score: i32, + pub filename_bonus: i32, + pub special_filename_bonus: i32, + pub frecency_boost: i32, + pub distance_penalty: i32, + pub current_file_penalty: i32, + pub combo_match_boost: i32, + pub path_alignment_bonus: i32, + pub exact_match: bool, + pub match_type: *mut c_char, +} + +impl From<&Score> for FffScore { + fn from(score: &Score) -> Self { + FffScore { + total: score.total, + base_score: score.base_score, + filename_bonus: score.filename_bonus, + special_filename_bonus: score.special_filename_bonus, + frecency_boost: score.frecency_boost, + distance_penalty: score.distance_penalty, + current_file_penalty: score.current_file_penalty, + combo_match_boost: score.combo_match_boost, + path_alignment_bonus: score.path_alignment_bonus, + exact_match: score.exact_match, + match_type: cstring_new(score.match_type), + } + } +} + +impl FffScore { + /// ## Safety + /// `match_type` must have been allocated by `CString::into_raw`. + pub unsafe fn free_strings(&mut self) { + unsafe { + if !self.match_type.is_null() { + drop(CString::from_raw(self.match_type)); + } + } + } +} + +/// Location parsed from a query string (e.g. `"file.ts:42:10"`). +/// +/// `tag` encodes the variant: +/// 0 = no location, +/// 1 = line only (`line` is set), +/// 2 = position (`line` + `col`), +/// 3 = range (`line`/`col` = start, `end_line`/`end_col` = end). +#[repr(C)] +pub struct FffLocation { + pub tag: u8, + pub line: i32, + pub col: i32, + pub end_line: i32, + pub end_col: i32, +} + +impl From> for FffLocation { + fn from(loc: Option<&Location>) -> Self { + match loc { + None => FffLocation { + tag: 0, + line: 0, + col: 0, + end_line: 0, + end_col: 0, + }, + Some(Location::Line(line)) => FffLocation { + tag: 1, + line: *line, + col: 0, + end_line: 0, + end_col: 0, + }, + Some(Location::Position { line, col }) => FffLocation { + tag: 2, + line: *line, + col: *col, + end_line: 0, + end_col: 0, + }, + Some(Location::Range { start, end }) => FffLocation { + tag: 3, + line: start.0, + col: start.1, + end_line: end.0, + end_col: end.1, + }, + } + } +} + +/// Search result returned by `fff_search`. +/// +/// The caller must free this with `fff_free_search_result`. +#[repr(C)] +pub struct FffSearchResult { + /// Pointer to a heap-allocated array of `FffFileItem` (length = `count`). + pub items: *mut FffFileItem, + /// Pointer to a heap-allocated array of `FffScore` (length = `count`). + pub scores: *mut FffScore, + /// Number of items/scores in the arrays. + pub count: u32, + /// Total number of files that matched the query. + pub total_matched: u32, + /// Total number of indexed files. + pub total_files: u32, + /// Location parsed from the query string. + pub location: FffLocation, +} + +impl FffSearchResult { + /// Convert a core `SearchResult` into a heap-allocated `FffSearchResult`. + pub fn from_core(result: &SearchResult, picker: &FilePicker) -> *mut Self { + let items: Vec = result + .items + .iter() + .map(|i| FffFileItem::from_item(i, picker)) + .collect(); + let scores: Vec = result.scores.iter().map(FffScore::from).collect(); + let count = items.len() as u32; + + let (items_ptr, _) = vec_to_raw(items); + let (scores_ptr, _) = vec_to_raw(scores); + + Box::into_raw(Box::new(FffSearchResult { + items: items_ptr, + scores: scores_ptr, + count, + total_matched: result.total_matched as u32, + total_files: result.total_files as u32, + location: FffLocation::from(result.location.as_ref()), + })) + } +} + +// --------------------------------------------------------------------------- +// Grep result types +// --------------------------------------------------------------------------- + +/// A byte range within a matched line, used for highlighting. +#[repr(C)] +pub struct FffMatchRange { + pub start: u32, + pub end: u32, +} + +/// A single grep match with file and line information. +/// +/// All string fields and arrays are heap-allocated. Free the parent +/// `FffGrepResult` with `fff_free_grep_result` to release everything. +#[repr(C)] +pub struct FffGrepMatch { + // -- pointers (8 bytes each) -- + pub relative_path: *mut c_char, + pub file_name: *mut c_char, + pub git_status: *mut c_char, + pub line_content: *mut c_char, + pub match_ranges: *mut FffMatchRange, + pub context_before: *mut *mut c_char, + pub context_after: *mut *mut c_char, + // -- 8-byte numeric fields -- + pub size: u64, + pub modified: u64, + pub total_frecency_score: i64, + pub access_frecency_score: i64, + pub modification_frecency_score: i64, + pub line_number: u64, + pub byte_offset: u64, + // -- 4-byte fields -- + pub col: u32, + pub match_ranges_count: u32, + pub context_before_count: u32, + pub context_after_count: u32, + // -- 2-byte fields -- + pub fuzzy_score: u16, + // -- 1-byte fields -- + pub has_fuzzy_score: bool, + pub is_binary: bool, + pub is_definition: bool, +} + +impl FffGrepMatch { + fn from_core_with_file(m: &GrepMatch, file: &FileItem, picker: &FilePicker) -> Self { + let ranges: Vec = m + .match_byte_offsets + .iter() + .map(|&(start, end)| FffMatchRange { start, end }) + .collect(); + let (match_ranges, match_ranges_count) = vec_to_raw(ranges); + let (context_before, context_before_count) = strings_to_raw(&m.context_before); + let (context_after, context_after_count) = strings_to_raw(&m.context_after); + let (has_fuzzy_score, fuzzy_score) = match m.fuzzy_score { + Some(s) => (true, s), + None => (false, 0), + }; + + FffGrepMatch { + relative_path: cstring_new(&file.relative_path(picker)), + file_name: cstring_new(&file.file_name(picker)), + git_status: cstring_new(format_git_status(file.git_status)), + line_content: cstring_new(&m.line_content), + match_ranges, + context_before, + context_after, + size: file.size, + modified: file.modified, + total_frecency_score: file.total_frecency_score() as i64, + access_frecency_score: file.access_frecency_score as i64, + modification_frecency_score: file.modification_frecency_score as i64, + line_number: m.line_number, + byte_offset: m.byte_offset, + col: m.col as u32, + match_ranges_count, + context_before_count, + context_after_count, + fuzzy_score, + has_fuzzy_score, + is_binary: file.is_binary(), + is_definition: m.is_definition, + } + } + + /// ## Safety + /// All pointers must have been allocated by the corresponding `from_core`. + pub unsafe fn free_fields(&mut self) { + unsafe { + if !self.relative_path.is_null() { + drop(CString::from_raw(self.relative_path)); + } + if !self.file_name.is_null() { + drop(CString::from_raw(self.file_name)); + } + if !self.git_status.is_null() { + drop(CString::from_raw(self.git_status)); + } + if !self.line_content.is_null() { + drop(CString::from_raw(self.line_content)); + } + if !self.match_ranges.is_null() { + drop(Vec::from_raw_parts( + self.match_ranges, + self.match_ranges_count as usize, + self.match_ranges_count as usize, + )); + } + free_cstring_array(self.context_before, self.context_before_count); + free_cstring_array(self.context_after, self.context_after_count); + } + } +} + +/// Grep result returned by `fff_live_grep` and `fff_multi_grep`. +/// +/// The caller must free this with `fff_free_grep_result`. +#[repr(C)] +pub struct FffGrepResult { + /// Pointer to a heap-allocated array of `FffGrepMatch` (length = `count`). + pub items: *mut FffGrepMatch, + /// Number of matches in the `items` array. + pub count: u32, + /// Total number of matches (always equal to `count`). + pub total_matched: u32, + /// Number of files actually opened and searched in this call. + pub total_files_searched: u32, + /// Total number of indexed files (before any filtering). + pub total_files: u32, + /// Number of files eligible for search after filtering. + pub filtered_file_count: u32, + /// File offset for the next page. 0 if all files have been searched. + pub next_file_offset: u32, + /// Regex compilation error when falling back to literal matching. Null if none. + pub regex_fallback_error: *mut c_char, +} + +impl FffGrepResult { + /// Convert a core `GrepResult` into a heap-allocated `FffGrepResult`. + pub fn from_core(result: &GrepResult, picker: &FilePicker) -> *mut Self { + let items: Vec = result + .matches + .iter() + .map(|m| { + let file = result.files[m.file_index]; + FffGrepMatch::from_core_with_file(m, file, picker) + }) + .collect(); + let (items_ptr, count) = vec_to_raw(items); + + Box::into_raw(Box::new(FffGrepResult { + items: items_ptr, + count, + total_matched: result.matches.len() as u32, + total_files_searched: result.total_files_searched as u32, + total_files: result.total_files as u32, + filtered_file_count: result.filtered_file_count as u32, + next_file_offset: result.next_file_offset as u32, + regex_fallback_error: match &result.regex_fallback_error { + Some(e) => cstring_new(e), + None => ptr::null_mut(), + }, + })) + } +} + +/// Result envelope returned by all `fff_*` functions. +/// +/// Heap-allocated. The caller must free it with `fff_free_result`. Calling `fff_free_result` +/// **does not** deallocate the underlying `handle` pointer. It needs to be cleaned separately. +/// see (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`, `fff_free_string`, etc.). +/// +/// Depending on the function, the payload is delivered through different fields: +/// +/// | Function | Payload field | Type | +/// |----------------------------|---------------|-------------------------------| +/// | `fff_create_instance` | `handle` | opaque instance pointer | +/// | `fff_search` | `handle` | `*mut FffSearchResult` | +/// | `fff_live_grep` | `handle` | `*mut FffGrepResult` | +/// | `fff_multi_grep` | `handle` | `*mut FffGrepResult` | +/// | `fff_get_scan_progress` | `handle` | `*mut FffScanProgress` | +/// | `fff_health_check` | `handle` | `*mut c_char` (JSON string) | +/// | `fff_get_historical_query` | `handle` | `*mut c_char` (string or null)| +/// | `fff_wait_for_scan` | `int_value` | 1 = completed, 0 = timed out | +/// | `fff_track_query` | `int_value` | 1 = success, 0 = failure | +/// | `fff_refresh_git_status` | `int_value` | number of files updated | +/// | `fff_scan_files` | (none) | success flag only | +/// | `fff_restart_index` | (none) | success flag only | +/// +/// On failure, `success` is false and `error` contains the message. +#[repr(C)] +pub struct FffResult { + /// Whether the operation succeeded. + pub success: bool, + /// Error message on failure. Null on success. + pub error: *mut c_char, + /// Opaque pointer payload. May be null. + pub handle: *mut c_void, + /// Integer payload for simple return values (bool as 0/1, counts, etc.). + pub int_value: i64, +} + +impl FffResult { + /// Create a successful result with no payload, returned as heap pointer. + pub fn ok_empty() -> *mut Self { + Box::into_raw(Box::new(FffResult { + success: true, + error: ptr::null_mut(), + handle: ptr::null_mut(), + int_value: 0, + })) + } + + /// Create a successful result with an integer value. + pub fn ok_int(value: i64) -> *mut Self { + Box::into_raw(Box::new(FffResult { + success: true, + error: ptr::null_mut(), + handle: ptr::null_mut(), + int_value: value, + })) + } + + /// Create a successful result carrying an opaque pointer (handle, typed struct, or string). + pub fn ok_handle(handle: *mut c_void) -> *mut Self { + Box::into_raw(Box::new(FffResult { + success: true, + error: ptr::null_mut(), + handle, + int_value: 0, + })) + } + + /// Create a successful result carrying a C string in the `handle` field. + /// The caller must free it with `fff_free_string`. + pub fn ok_string(s: &str) -> *mut Self { + let cstr = CString::new(s).unwrap_or_default().into_raw(); + Box::into_raw(Box::new(FffResult { + success: true, + error: ptr::null_mut(), + handle: cstr as *mut c_void, + int_value: 0, + })) + } + + /// Create an error result, returned as heap pointer. + pub fn err(error: &str) -> *mut Self { + Box::into_raw(Box::new(FffResult { + success: false, + error: CString::new(error).unwrap_or_default().into_raw(), + handle: ptr::null_mut(), + int_value: 0, + })) + } +} + +/// A directory item returned by `fff_search_directories`. +/// +/// All string fields are heap-allocated and owned by the parent `FffDirSearchResult`. +/// Free the entire result with `fff_free_dir_search_result`. +#[repr(C)] +pub struct FffDirItem { + pub relative_path: *mut c_char, + pub dir_name: *mut c_char, + pub max_access_frecency: i32, +} + +impl FffDirItem { + pub fn from_item(item: &DirItem, picker: &FilePicker) -> Self { + FffDirItem { + relative_path: cstring_new(&item.relative_path(picker)), + dir_name: cstring_new(&item.dir_name(picker)), + max_access_frecency: item.max_access_frecency(), + } + } + + /// ## Safety + /// All string pointers must have been allocated by the rust side + pub unsafe fn free_strings(&mut self) { + unsafe { + if !self.relative_path.is_null() { + drop(CString::from_raw(self.relative_path)); + } + if !self.dir_name.is_null() { + drop(CString::from_raw(self.dir_name)); + } + } + } +} + +/// Directory search result returned by `fff_search_directories`. +/// +/// The caller must free this with `fff_free_dir_search_result`. +#[repr(C)] +pub struct FffDirSearchResult { + /// Pointer to a heap-allocated array of `FffDirItem` (length = `count`). + pub items: *mut FffDirItem, + /// Pointer to a heap-allocated array of `FffScore` (length = `count`). + pub scores: *mut FffScore, + /// Number of items/scores in the arrays. + pub count: u32, + /// Total number of directories that matched the query. + pub total_matched: u32, + /// Total number of indexed directories. + pub total_dirs: u32, +} + +impl FffDirSearchResult { + /// Convert a core `DirSearchResult` into a heap-allocated `FffDirSearchResult`. + pub fn from_core(result: &DirSearchResult, picker: &FilePicker) -> *mut Self { + let items: Vec = result + .items + .iter() + .map(|i| FffDirItem::from_item(i, picker)) + .collect(); + let scores: Vec = result.scores.iter().map(FffScore::from).collect(); + let count = items.len() as u32; + + let (items_ptr, _) = vec_to_raw(items); + let (scores_ptr, _) = vec_to_raw(scores); + + Box::into_raw(Box::new(FffDirSearchResult { + items: items_ptr, + scores: scores_ptr, + count, + total_matched: result.total_matched as u32, + total_dirs: result.total_dirs as u32, + })) + } +} + +/// A single item in a mixed (files + directories) search result. +/// +/// `item_type`: 0 = file, 1 = directory. +/// All string fields are heap-allocated and owned by the parent `FffMixedSearchResult`. +#[repr(C)] +pub struct FffMixedItem { + /// 0 = file, 1 = directory. + pub item_type: u8, + pub relative_path: *mut c_char, + /// Filename for files, last directory segment for directories. + pub display_name: *mut c_char, + pub git_status: *mut c_char, + pub size: u64, + pub modified: u64, + /// The access frecency score for files, or max access frecency among all the immediate + /// children for directories. + pub access_frecency_score: i64, + /// Always 0 for directories + pub modification_frecency_score: i64, + /// Always 0 for directories + pub total_frecency_score: i64, + /// Always 0 for directories + pub is_binary: bool, +} + +impl FffMixedItem { + pub fn from_mixed_ref(item: &MixedItemRef<'_>, picker: &FilePicker) -> Self { + match item { + MixedItemRef::File(file) => FffMixedItem { + item_type: 0, + relative_path: cstring_new(&file.relative_path(picker)), + display_name: cstring_new(&file.file_name(picker)), + git_status: cstring_new(format_git_status(file.git_status)), + size: file.size, + modified: file.modified, + access_frecency_score: file.access_frecency_score as i64, + modification_frecency_score: file.modification_frecency_score as i64, + total_frecency_score: file.total_frecency_score() as i64, + is_binary: file.is_binary(), + }, + MixedItemRef::Dir(dir) => FffMixedItem { + item_type: 1, + relative_path: cstring_new(&dir.relative_path(picker)), + display_name: cstring_new(&dir.dir_name(picker)), + git_status: cstring_new(""), + size: 0, + modified: 0, + access_frecency_score: dir.max_access_frecency() as i64, + modification_frecency_score: 0, + total_frecency_score: dir.max_access_frecency() as i64, + is_binary: false, + }, + } + } + + /// ## Safety + /// All string pointers must have been allocated by rust side + pub unsafe fn free_strings(&mut self) { + unsafe { + if !self.relative_path.is_null() { + drop(CString::from_raw(self.relative_path)); + } + if !self.display_name.is_null() { + drop(CString::from_raw(self.display_name)); + } + if !self.git_status.is_null() { + drop(CString::from_raw(self.git_status)); + } + } + } +} + +/// Mixed search result returned by `fff_search_mixed`. +/// +/// The caller must free this with `fff_free_mixed_search_result`. +#[repr(C)] +pub struct FffMixedSearchResult { + /// Pointer to a heap-allocated array of `FffMixedItem` (length = `count`). + pub items: *mut FffMixedItem, + /// Pointer to a heap-allocated array of `FffScore` (length = `count`). + pub scores: *mut FffScore, + /// Number of items/scores in the arrays. + pub count: u32, + /// Total number of items (files + dirs) that matched the query. + pub total_matched: u32, + /// Total number of indexed files. + pub total_files: u32, + /// Total number of indexed directories. + pub total_dirs: u32, + /// Location parsed from the query string. + pub location: FffLocation, +} + +impl FffMixedSearchResult { + /// Convert a core `MixedSearchResult` into a heap-allocated `FffMixedSearchResult`. + pub fn from_core(result: &MixedSearchResult, picker: &FilePicker) -> *mut Self { + let items: Vec = result + .items + .iter() + .map(|i| FffMixedItem::from_mixed_ref(i, picker)) + .collect(); + let scores: Vec = result.scores.iter().map(FffScore::from).collect(); + let count = items.len() as u32; + + let (items_ptr, _) = vec_to_raw(items); + let (scores_ptr, _) = vec_to_raw(scores); + + Box::into_raw(Box::new(FffMixedSearchResult { + items: items_ptr, + scores: scores_ptr, + count, + total_matched: result.total_matched as u32, + total_files: result.total_files as u32, + total_dirs: result.total_dirs as u32, + location: FffLocation::from(result.location.as_ref()), + })) + } +} + +/// Scan progress returned by `fff_get_scan_progress`. +/// The caller must free this with `fff_free_scan_progress`. +#[repr(C)] +pub struct FffScanProgress { + pub scanned_files_count: u64, + pub is_scanning: bool, + pub is_watcher_ready: bool, + pub is_warmup_complete: bool, +} + +impl From for FffScanProgress { + fn from(p: fff::file_picker::ScanProgress) -> Self { + Self { + scanned_files_count: p.scanned_files_count as u64, + is_scanning: p.is_scanning, + is_watcher_ready: p.is_watcher_ready, + is_warmup_complete: p.is_warmup_complete, + } + } +} + +#[cfg(test)] +mod options_layout_tests { + use super::FffCreateOptions; + use std::mem::{align_of, offset_of, size_of}; + + // THIS TEST HAVE TO BE NEVER UPDATED ONLY ADDED NEW FIELDS + // this is needed to ensure ABI backward compatibility + #[test] + #[cfg(target_pointer_width = "64")] + fn fff_create_options_layout_is_stable_64bit() { + assert_eq!(size_of::(), 88); + assert_eq!(align_of::(), 8); + + assert_eq!(offset_of!(FffCreateOptions, version), 0); + assert_eq!(offset_of!(FffCreateOptions, base_path), 8); + assert_eq!(offset_of!(FffCreateOptions, frecency_db_path), 16); + assert_eq!(offset_of!(FffCreateOptions, history_db_path), 24); + assert_eq!(offset_of!(FffCreateOptions, enable_mmap_cache), 32); + assert_eq!(offset_of!(FffCreateOptions, enable_content_indexing), 33); + assert_eq!(offset_of!(FffCreateOptions, watch), 34); + assert_eq!(offset_of!(FffCreateOptions, ai_mode), 35); + assert_eq!(offset_of!(FffCreateOptions, log_file_path), 40); + assert_eq!(offset_of!(FffCreateOptions, log_level), 48); + assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_files), 56); + assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_bytes), 64); + assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_file_size), 72); + assert_eq!(offset_of!(FffCreateOptions, enable_fs_root_scanning), 80); + assert_eq!(offset_of!(FffCreateOptions, enable_home_dir_scanning), 81); + assert_eq!(offset_of!(FffCreateOptions, follow_symlinks), 82); + } +} diff --git a/crates/fff-c/src/lib.rs b/crates/fff-c/src/lib.rs new file mode 100644 index 0000000..56283d7 --- /dev/null +++ b/crates/fff-c/src/lib.rs @@ -0,0 +1,1699 @@ +//! C FFI bindings for fff-core +//! +//! This crate provides C-compatible FFI exports that can be used from any language +//! with C FFI support (Bun, Node.js, Python, Ruby, etc.). +//! +//! # Instance-based API +//! +//! All state is owned by an opaque `FffInstance` fff_handle. Callers create an instance +//! with `fff_create_instance`, pass the fff_handle to every subsequent call, and free it with +//! `fff_destroy`. Multiple independent instances can coexist in the same process. +//! +//! # Memory management +//! +//! * Every `fff_*` function that returns `*mut FffResult` requires the caller to +//! free the result with `fff_free_result`. +//! * The instance itself must be freed with `fff_destroy`. +//! +//! # Parameter conventions +//! +//! * Optional `*const c_char` parameters: pass NULL or an empty string to omit. +//! * Numeric parameters: 0 means "use default" unless documented otherwise. +//! * Grep mode (`u8`): 0 = plain text, 1 = regex, 2 = fuzzy. +//! * Multi-grep patterns are passed as a single newline-separated (`\n`) string. + +use std::ffi::{CStr, CString, c_char, c_void}; +use std::path::PathBuf; +use std::time::Duration; + +use fff::shared::SharedQueryTracker; + +mod accessors; +mod ffi_types; + +use fff::file_picker::FilePicker; +use fff::frecency::FrecencyTracker; +use fff::query_tracker::QueryTracker; +use fff::{DbHealthChecker, FFFMode, FuzzySearchOptions, PaginationArgs, QueryParser}; +use fff::{SharedFilePicker, SharedFrecency}; +use ffi_types::{ + FFF_CREATE_OPTIONS_VERSION, FffCreateOptions, FffDirItem, FffDirSearchResult, FffFileItem, + FffGrepMatch, FffGrepResult, FffMixedItem, FffMixedSearchResult, FffResult, FffScanProgress, + FffScore, FffSearchResult, +}; + +/// Opaque fff_handle holding all per-instance state. +/// +/// The caller receives this as `*mut c_void` and must pass it to every FFI call. +/// The fff_handle is freed by `fff_destroy`. +struct FffInstance { + picker: SharedFilePicker, + frecency: SharedFrecency, + query_tracker: SharedQueryTracker, +} + +/// Helper to convert C string to Rust &str. +/// +/// Returns `None` if the pointer is null or the string is not valid UTF-8. +unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> { + if s.is_null() { + None + } else { + unsafe { CStr::from_ptr(s).to_str().ok() } + } +} + +/// Helper to convert an optional C string parameter. +/// +/// Returns `None` if the pointer is null, empty, or not valid UTF-8. +unsafe fn optional_cstr<'a>(s: *const c_char) -> Option<&'a str> { + unsafe { cstr_to_str(s) }.filter(|s| !s.is_empty()) +} + +/// Recover a `&FffInstance` from the opaque pointer. +/// +/// Returns an error `FffResult` if the pointer is null. +unsafe fn instance_ref<'a>(fff_handle: *mut c_void) -> Result<&'a FffInstance, *mut FffResult> { + if fff_handle.is_null() { + Err(FffResult::err( + "Instance handle is null. Create one with fff_create_instance first.", + )) + } else { + Ok(unsafe { &*(fff_handle as *const FffInstance) }) + } +} + +/// Decode a `u8` grep mode into the core enum. +fn grep_mode_from_u8(mode: u8) -> fff::GrepMode { + match mode { + 1 => fff::GrepMode::Regex, + 2 => fff::GrepMode::Fuzzy, + _ => fff::GrepMode::PlainText, + } +} + +/// Apply "0 means default" convention. +fn default_u32(val: u32, default: u32) -> u32 { + if val == 0 { default } else { val } +} + +fn default_u64(val: u64, default: u64) -> u64 { + if val == 0 { default } else { val } +} + +fn default_i32(val: i32, default: i32) -> i32 { + if val == 0 { default } else { val } +} + +/// Create a new file finder instance (legacy 8-arg positional signature). +/// +/// @deprecated Use [`fff_create_instance_with`] (or +/// [`fff_create_instance_with_value`] for FFI bindings) — both take the +/// versioned [`FffCreateOptions`] struct that evolves without ABI breaks. +/// This function delegates to `fff_create_instance_with` internally; the +/// `use_unsafe_no_lock` parameter is deprecated and ignored. +/// +/// ## Safety +/// See `fff_create_instance_with`. +#[deprecated( + since = "0.8.5", + note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks." +)] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_create_instance( + base_path: *const c_char, + frecency_db_path: *const c_char, + history_db_path: *const c_char, + _use_unsafe_no_lock: bool, + enable_mmap_cache: bool, + enable_content_indexing: bool, + watch: bool, + ai_mode: bool, +) -> *mut FffResult { + let mut opts = FffCreateOptions::defaults(); + opts.base_path = base_path; + opts.frecency_db_path = frecency_db_path; + opts.history_db_path = history_db_path; + opts.enable_mmap_cache = enable_mmap_cache; + opts.enable_content_indexing = enable_content_indexing; + opts.watch = watch; + opts.ai_mode = ai_mode; + unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) } +} + +/// Create a new file finder instance (legacy 13-arg positional signature). +/// +/// @deprecated Use [`fff_create_instance_with`] (or +/// [`fff_create_instance_with_value`] for FFI bindings) — both take the +/// versioned [`FffCreateOptions`] struct that evolves without ABI breaks. +/// The `use_unsafe_no_lock` parameter is deprecated and ignored. +/// +/// ## Safety +/// See `fff_create_instance_with`. +#[deprecated( + since = "0.8.5", + note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks." +)] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_create_instance2( + base_path: *const c_char, + frecency_db_path: *const c_char, + history_db_path: *const c_char, + _use_unsafe_no_lock: bool, + enable_mmap_cache: bool, + enable_content_indexing: bool, + watch: bool, + ai_mode: bool, + log_file_path: *const c_char, + log_level: *const c_char, + cache_budget_max_files: u64, + cache_budget_max_bytes: u64, + cache_budget_max_file_size: u64, +) -> *mut FffResult { + let mut opts = FffCreateOptions::defaults(); + opts.base_path = base_path; + opts.frecency_db_path = frecency_db_path; + opts.history_db_path = history_db_path; + opts.enable_mmap_cache = enable_mmap_cache; + opts.enable_content_indexing = enable_content_indexing; + opts.watch = watch; + opts.ai_mode = ai_mode; + opts.log_file_path = log_file_path; + opts.log_level = log_level; + opts.cache_budget_max_files = cache_budget_max_files; + opts.cache_budget_max_bytes = cache_budget_max_bytes; + opts.cache_budget_max_file_size = cache_budget_max_file_size; + unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) } +} + +/// Create a new file finder instance from an [`FffCreateOptions`] struct. +/// +/// **Direct C consumers** populate the struct (designated initializers +/// recommended), set `version` to [`FFF_CREATE_OPTIONS_VERSION`], and pass +/// it by pointer. New fields are appended in future versions; old callers +/// passing `version = 1` keep working forever. +/// +/// **FFI consumers** that prefer struct-by-value semantics (e.g. ffi-rs's +/// `paramsType: [structDef]`) should use [`fff_create_instance_with_value`] +/// instead — it's a thin calling-convention adapter that delegates here. +/// +/// Required: `opts.base_path` must be non-NULL and non-empty. +/// +/// When all three `cache_budget_*` values are 0 the budget is auto-computed +/// from repo size after the initial scan. Otherwise an explicit budget is +/// used: any field left at 0 falls back to its `unlimited()` default. +/// +/// ## Safety +/// * `opts` must be a valid pointer to an `FffCreateOptions` whose `version` +/// is in the range `1..=FFF_CREATE_OPTIONS_VERSION`. +/// * All string pointers inside `opts` must be valid null-terminated UTF-8 +/// or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_create_instance_with(opts: *const FffCreateOptions) -> *mut FffResult { + if opts.is_null() { + return FffResult::err("opts is null"); + } + let opts = unsafe { &*opts }; + if opts.version == 0 || opts.version > FFF_CREATE_OPTIONS_VERSION { + return FffResult::err(&format!( + "Unsupported FffCreateOptions version {} (library understands up to {})", + opts.version, FFF_CREATE_OPTIONS_VERSION + )); + } + + let base_path_str = match unsafe { cstr_to_str(opts.base_path) } { + Some(s) if !s.is_empty() => s.to_string(), + _ => return FffResult::err("opts.base_path is null or empty"), + }; + + if let Some(log_path) = unsafe { optional_cstr(opts.log_file_path) } { + let level = unsafe { optional_cstr(opts.log_level) }; + if let Err(e) = fff::log::init_tracing(log_path, level, None) { + return FffResult::err(&format!("Failed to init tracing: {}", e)); + } + } + + let frecency_path = unsafe { optional_cstr(opts.frecency_db_path) }.map(|s| s.to_string()); + let history_path = unsafe { optional_cstr(opts.history_db_path) }.map(|s| s.to_string()); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + let query_tracker = SharedQueryTracker::default(); + + if let Some(ref frecency_path) = frecency_path { + if let Some(parent) = PathBuf::from(frecency_path).parent() { + let _ = std::fs::create_dir_all(parent); + } + + match FrecencyTracker::open(frecency_path) { + Ok(tracker) => { + if let Err(e) = shared_frecency.init(tracker) { + return FffResult::err(&format!("Failed to acquire frecency lock: {}", e)); + } + } + Err(e) => return FffResult::err(&format!("Failed to init frecency db: {}", e)), + } + } + + if let Some(ref history_path) = history_path { + if let Some(parent) = PathBuf::from(history_path).parent() { + let _ = std::fs::create_dir_all(parent); + } + + match QueryTracker::open(history_path) { + Ok(tracker) => { + if let Err(e) = query_tracker.init(tracker) { + return FffResult::err(&format!("Failed to acquire query tracker lock: {}", e)); + } + } + Err(e) => return FffResult::err(&format!("Failed to init query tracker db: {}", e)), + } + } + + let mode = if opts.ai_mode { + FFFMode::Ai + } else { + FFFMode::Neovim + }; + + let cache_budget = fff::ContentCacheBudget::from_overrides( + opts.cache_budget_max_files as usize, + opts.cache_budget_max_bytes, + opts.cache_budget_max_file_size, + ); + + if let Err(e) = FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + fff::FilePickerOptions { + base_path: base_path_str, + enable_mmap_cache: opts.enable_mmap_cache, + enable_content_indexing: opts.enable_content_indexing, + watch: opts.watch, + mode, + cache_budget, + follow_symlinks: opts.version >= 2 && opts.follow_symlinks, + enable_fs_root_scanning: opts.enable_fs_root_scanning, + enable_home_dir_scanning: opts.enable_home_dir_scanning, + }, + ) { + return FffResult::err(&format!("Failed to init file picker: {}", e)); + } + + let instance = Box::new(FffInstance { + picker: shared_picker, + frecency: shared_frecency, + query_tracker, + }); + + let fff_handle = Box::into_raw(instance) as *mut c_void; + FffResult::ok_handle(fff_handle) +} + +/// Calling-convention adapter for [`fff_create_instance_with`]. +/// +/// Same logic, but takes the [`FffCreateOptions`] struct **by value**. This +/// makes the function callable from FFI libraries whose native struct +/// support passes structs by value on the wire (e.g. Node's `ffi-rs` with +/// `paramsType: [structDef]`). +/// +/// This is **not** a versioned wrapper — when new fields are appended to +/// `FffCreateOptions`, both this function and `fff_create_instance_with` +/// pick them up automatically with no signature change. +/// +/// ## Safety +/// All `*const c_char` fields inside `opts` must be valid null-terminated +/// UTF-8 or NULL. The struct itself is consumed by value. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_create_instance_with_value(opts: FffCreateOptions) -> *mut FffResult { + unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) } +} + +/// Destroy a file finder instance and free all its resources. +/// +/// ## Safety +/// `fff_handle` must be a valid pointer returned by `fff_create_instance`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) { + if fff_handle.is_null() { + return; + } + + let instance = unsafe { Box::from_raw(fff_handle as *mut FffInstance) }; + + if let Ok(mut guard) = instance.picker.write() + && let Some(picker) = guard.take() + { + drop(picker); + } + + if let Ok(mut guard) = instance.frecency.write() { + *guard = None; + } + if let Ok(mut guard) = instance.query_tracker.write() { + *guard = None; + } +} + +/// Perform fuzzy search on indexed files. +/// +/// # Parameters +/// +/// * `fff_handle` – instance from `fff_create_instance` +/// * `query` – search query string +/// * `current_file` – path of the currently open file for deprioritization (NULL/empty to skip) +/// * `max_threads` – maximum worker threads (0 = auto-detect) +/// * `page_index` – pagination offset (0 = first page) +/// * `page_size` – results per page (0 = default 100) +/// * `combo_boost_multiplier` – score multiplier for combo matches (0 = default 100) +/// * `min_combo_count` – minimum combo count before boost applies (0 = default 3) +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search( + fff_handle: *mut c_void, + query: *const c_char, + current_file: *const c_char, + max_threads: u32, + page_index: u32, + page_size: u32, + combo_boost_multiplier: i32, + min_combo_count: u32, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let query_str = match unsafe { cstr_to_str(query) } { + Some(s) => s, + None => return FffResult::err("Query is null or invalid UTF-8"), + }; + + let current_file_str = unsafe { optional_cstr(current_file) }; + let page_size = default_u32(page_size, 100) as usize; + let min_combo_count = default_u32(min_combo_count, 3); + let combo_boost_multiplier = default_i32(combo_boost_multiplier, 100); + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + // Get query tracker ref for combo matching + let qt_guard = match inst.query_tracker.read() { + Ok(q) => q, + Err(_) => return FffResult::err("Failed to acquire query tracker lock"), + }; + let query_tracker_ref = qt_guard.as_ref(); + + let parser = QueryParser::default(); + let parsed = parser.parse(query_str); + + let results = picker.fuzzy_search( + &parsed, + query_tracker_ref, + FuzzySearchOptions { + max_threads: max_threads as usize, + current_file: current_file_str, + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: combo_boost_multiplier, + min_combo_count, + pagination: PaginationArgs { + offset: page_index as usize, + limit: page_size, + }, + }, + ); + + let search_result = FffSearchResult::from_core(&results, picker); + FffResult::ok_handle(search_result as *mut c_void) +} + +/// Glob-only search: filter indexed files by a single glob pattern, rank by +/// frecency, and paginate. Bypasses the regular query parser entirely. +/// +/// Use this when you already have a literal glob pattern (e.g. `*.rs`, a +/// recursive `**` match, or `src/components` prefix) and want neither fuzzy +/// matching nor multi-token constraint parsing. Ranking falls back to +/// frecency because there is no fuzzy score to combine with. +/// +/// # Parameters +/// +/// * `fff_handle` - instance from `fff_create_instance` +/// * `pattern` - glob pattern (required, no parsing - passed through verbatim) +/// * `current_file` - path of the currently open file for deprioritization (NULL/empty to skip) +/// * `max_threads` - maximum worker threads (0 = auto-detect) +/// * `page_index` - pagination offset (0 = first page) +/// * `page_size` - results per page (0 = default 100) +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `pattern` and `current_file` must be valid null-terminated UTF-8 strings or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_glob( + fff_handle: *mut c_void, + pattern: *const c_char, + current_file: *const c_char, + max_threads: u32, + page_index: u32, + page_size: u32, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let pattern_str = match unsafe { cstr_to_str(pattern) } { + Some(s) if !s.is_empty() => s, + _ => return FffResult::err("Pattern is null, empty, or invalid UTF-8"), + }; + + let current_file_str = unsafe { optional_cstr(current_file) }; + let page_size = default_u32(page_size, 100) as usize; + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + let results = picker.glob( + pattern_str, + FuzzySearchOptions { + max_threads: max_threads as usize, + current_file: current_file_str, + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: 0, + min_combo_count: 0, + pagination: PaginationArgs { + offset: page_index as usize, + limit: page_size, + }, + }, + ); + + let search_result = FffSearchResult::from_core(&results, picker); + FffResult::ok_handle(search_result as *mut c_void) +} + +/// Perform fuzzy search on indexed directories. +/// +/// # Parameters +/// +/// * `fff_handle` – instance from `fff_create_instance` +/// * `query` – search query string +/// * `current_file` – path of the currently open file for distance scoring (NULL/empty to skip) +/// * `max_threads` – maximum worker threads (0 = auto-detect) +/// * `page_index` – pagination offset (0 = first page) +/// * `page_size` – results per page (0 = default 100) +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_directories( + fff_handle: *mut c_void, + query: *const c_char, + current_file: *const c_char, + max_threads: u32, + page_index: u32, + page_size: u32, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let query_str = match unsafe { cstr_to_str(query) } { + Some(s) => s, + None => return FffResult::err("Query is null or invalid UTF-8"), + }; + + let current_file_str = unsafe { optional_cstr(current_file) }; + let page_size = default_u32(page_size, 100) as usize; + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + let parser = QueryParser::new(fff_query_parser::DirSearchConfig); + let parsed = parser.parse(query_str); + + let results = picker.fuzzy_search_directories( + &parsed, + FuzzySearchOptions { + max_threads: max_threads as usize, + current_file: current_file_str, + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: 0, + min_combo_count: 0, + pagination: PaginationArgs { + offset: page_index as usize, + limit: page_size, + }, + }, + ); + + let dir_result = FffDirSearchResult::from_core(&results, picker); + FffResult::ok_handle(dir_result as *mut c_void) +} + +/// Perform a mixed fuzzy search across both files and directories. +/// +/// Returns a single flat list where files and directories are interleaved +/// by total score in descending order. Each item has an `item_type` field +/// (0 = file, 1 = directory). +/// +/// # Parameters +/// +/// * `fff_handle` – instance from `fff_create_instance` +/// * `query` – search query string +/// * `current_file` – path of the currently open file (NULL/empty to skip) +/// * `max_threads` – maximum worker threads (0 = auto-detect) +/// * `page_index` – pagination offset (0 = first page) +/// * `page_size` – results per page (0 = default 100) +/// * `combo_boost_multiplier` – score multiplier for combo matches (0 = default 100) +/// * `min_combo_count` – minimum combo count before boost applies (0 = default 3) +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `query` and `current_file` must be valid null-terminated UTF-8 strings or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_mixed( + fff_handle: *mut c_void, + query: *const c_char, + current_file: *const c_char, + max_threads: u32, + page_index: u32, + page_size: u32, + combo_boost_multiplier: i32, + min_combo_count: u32, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let query_str = match unsafe { cstr_to_str(query) } { + Some(s) => s, + None => return FffResult::err("Query is null or invalid UTF-8"), + }; + + let current_file_str = unsafe { optional_cstr(current_file) }; + let page_size = default_u32(page_size, 100) as usize; + let min_combo_count = default_u32(min_combo_count, 3); + let combo_boost_multiplier = default_i32(combo_boost_multiplier, 100); + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + let qt_guard = match inst.query_tracker.read() { + Ok(q) => q, + Err(_) => return FffResult::err("Failed to acquire query tracker lock"), + }; + let query_tracker_ref = qt_guard.as_ref(); + + let parser = QueryParser::new(fff_query_parser::MixedSearchConfig); + let parsed = parser.parse(query_str); + + let results = picker.fuzzy_search_mixed( + &parsed, + query_tracker_ref, + FuzzySearchOptions { + max_threads: max_threads as usize, + current_file: current_file_str, + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: combo_boost_multiplier, + min_combo_count, + pagination: PaginationArgs { + offset: page_index as usize, + limit: page_size, + }, + }, + ); + + let mixed_result = FffMixedSearchResult::from_core(&results, picker); + FffResult::ok_handle(mixed_result as *mut c_void) +} + +/// Perform content search (grep) across indexed files. +/// +/// # Parameters +/// +/// * `fff_handle` – instance from `fff_create_instance` +/// * `query` – search query (supports constraint syntax like `*.rs pattern`) +/// * `mode` – 0 = plain text (SIMD), 1 = regex, 2 = fuzzy +/// * `max_file_size` – skip files larger than this in bytes (0 = default 10 MB) +/// * `max_matches_per_file` – max matches per file (0 = unlimited) +/// * `smart_case` – case-insensitive when query is all lowercase +/// * `file_offset` – file-based pagination offset (0 = start) +/// * `page_limit` – max matches to return (0 = default 50) +/// * `time_budget_ms` – wall-clock budget in ms (0 = unlimited) +/// * `before_context` – context lines before each match +/// * `after_context` – context lines after each match +/// * `classify_definitions` – tag matches that are code definitions +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `query` must be a valid null-terminated UTF-8 string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_live_grep( + fff_handle: *mut c_void, + query: *const c_char, + mode: u8, + max_file_size: u64, + max_matches_per_file: u32, + smart_case: bool, + file_offset: u32, + page_limit: u32, + time_budget_ms: u64, + before_context: u32, + after_context: u32, + classify_definitions: bool, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let query_str = match unsafe { cstr_to_str(query) } { + Some(s) => s, + None => return FffResult::err("Query is null or invalid UTF-8"), + }; + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + let is_ai = picker.mode().is_ai(); + let parsed = if is_ai { + fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse(query_str) + } else { + fff::grep::parse_grep_query(query_str) + }; + + let options = fff::GrepSearchOptions { + max_file_size: default_u64(max_file_size, 10 * 1024 * 1024), + max_matches_per_file: max_matches_per_file as usize, + smart_case, + file_offset: file_offset as usize, + page_limit: default_u32(page_limit, 50) as usize, + mode: grep_mode_from_u8(mode), + time_budget_ms, + before_context: before_context as usize, + after_context: after_context as usize, + classify_definitions, + trim_whitespace: false, + abort_signal: None, + }; + + let result = picker.grep(&parsed, &options); + let grep_result = FffGrepResult::from_core(&result, picker); + FffResult::ok_handle(grep_result as *mut c_void) +} + +/// Perform multi-pattern OR search (Aho-Corasick) across indexed files. +/// +/// Searches for lines matching ANY of the provided patterns using +/// SIMD-accelerated multi-needle matching. +/// +/// # Parameters +/// +/// * `fff_handle` – instance from `fff_create_instance` +/// * `patterns_joined` – patterns separated by `\n` (e.g. `"foo\nbar\nbaz"`) +/// * `constraints` – file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip) +/// * `max_file_size` – skip files larger than this in bytes (0 = default 10 MB) +/// * `max_matches_per_file` – max matches per file (0 = unlimited) +/// * `smart_case` – case-insensitive when all patterns are lowercase +/// * `file_offset` – file-based pagination offset (0 = start) +/// * `page_limit` – max matches to return (0 = default 50) +/// * `time_budget_ms` – wall-clock budget in ms (0 = unlimited) +/// * `before_context` – context lines before each match +/// * `after_context` – context lines after each match +/// * `classify_definitions` – tag matches that are code definitions +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `patterns_joined` and `constraints` must be valid null-terminated UTF-8 or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_multi_grep( + fff_handle: *mut c_void, + patterns_joined: *const c_char, + constraints: *const c_char, + max_file_size: u64, + max_matches_per_file: u32, + smart_case: bool, + file_offset: u32, + page_limit: u32, + time_budget_ms: u64, + before_context: u32, + after_context: u32, + classify_definitions: bool, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let patterns_str = match unsafe { cstr_to_str(patterns_joined) } { + Some(s) if !s.is_empty() => s, + _ => return FffResult::err("patterns_joined is null or empty"), + }; + + let patterns: Vec<&str> = patterns_str.split('\n').collect(); + if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) { + return FffResult::err("patterns must not be empty"); + } + + let constraints_str = unsafe { optional_cstr(constraints) }; + + let picker_guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match picker_guard.as_ref() { + Some(p) => p, + None => { + return FffResult::err("File picker not initialized. Call fff_create_instance first."); + } + }; + + let is_ai = picker.mode().is_ai(); + + // Parse constraints from the optional string (e.g. "*.rs /src/") + let parsed_constraints = constraints_str.map(|c| { + if is_ai { + fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse(c) + } else { + fff::grep::parse_grep_query(c) + } + }); + + let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints { + Some(q) => &q.constraints, + None => &[], + }; + + let options = fff::GrepSearchOptions { + max_file_size: default_u64(max_file_size, 10 * 1024 * 1024), + max_matches_per_file: max_matches_per_file as usize, + smart_case, + file_offset: file_offset as usize, + page_limit: default_u32(page_limit, 50) as usize, + mode: fff::GrepMode::PlainText, // ignored by multi_grep_search + time_budget_ms, + before_context: before_context as usize, + after_context: after_context as usize, + classify_definitions, + trim_whitespace: false, + abort_signal: None, + }; + + let result = picker.multi_grep(&patterns, constraint_refs, &options); + let grep_result = FffGrepResult::from_core(&result, picker); + FffResult::ok_handle(grep_result as *mut c_void) +} + +/// Trigger a rescan of the file index. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_scan_files(fff_handle: *mut c_void) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + // Async: rescan runs on a BG thread, caller returns immediately. + // Use `fff_is_scanning` / `fff_wait_for_scan` to observe progress. + match inst.picker.trigger_full_rescan_async(&inst.frecency) { + Ok(()) => FffResult::ok_empty(), + Err(e) => FffResult::err(&format!("Failed to trigger rescan: {}", e)), + } +} + +/// Check if a scan is currently in progress. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_is_scanning(fff_handle: *mut c_void) -> bool { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(_) => return false, + }; + + inst.picker + .read() + .ok() + .and_then(|guard| guard.as_ref().map(|p| p.is_scan_active())) + .unwrap_or(false) +} + +/// Get the base path of the file picker. +/// +/// Returns an `FffResult` with a heap-allocated C string in the `handle` +/// field. Free the string with `fff_free_string` after reading it. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_get_base_path(fff_handle: *mut c_void) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match guard.as_ref() { + Some(p) => p, + None => return FffResult::err("File picker not initialized"), + }; + + FffResult::ok_string(&picker.base_path().to_string_lossy()) +} + +/// Get scan progress information. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_get_scan_progress(fff_handle: *mut c_void) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let guard = match inst.picker.read() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let picker = match guard.as_ref() { + Some(p) => p, + None => return FffResult::err("File picker not initialized"), + }; + + let result = Box::into_raw(Box::new(FffScanProgress::from(picker.get_scan_progress()))); + FffResult::ok_handle(result as *mut c_void) +} + +/// Wait for initial scan to complete. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_wait_for_scan( + fff_handle: *mut c_void, + timeout_ms: u64, +) -> *mut FffResult { + let FffInstance { picker, .. } = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let completed = picker.wait_for_scan(Duration::from_millis(timeout_ms)); + FffResult::ok_int(completed as i64) +} + +/// Wait for the background file watcher to be ready. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_wait_for_watcher( + fff_handle: *mut c_void, + timeout_ms: u64, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let completed = inst + .picker + .wait_for_watcher(Duration::from_millis(timeout_ms)); + FffResult::ok_int(completed as i64) +} + +/// Restart indexing in a new directory. +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `new_path` must be a valid null-terminated UTF-8 string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_restart_index( + fff_handle: *mut c_void, + new_path: *const c_char, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let path_str = match unsafe { cstr_to_str(new_path) } { + Some(s) => s, + None => return FffResult::err("Path is null or invalid UTF-8"), + }; + + let path = PathBuf::from(&path_str); + if !path.exists() { + return FffResult::err(&format!("Path does not exist: {}", path_str)); + } + + let canonical_path = match fff::path_utils::canonicalize(&path) { + Ok(p) => p, + Err(e) => return FffResult::err(&format!("Failed to canonicalize path: {}", e)), + }; + + let guard = match inst.picker.write() { + Ok(g) => g, + Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)), + }; + + let (warmup_caches, content_indexing, watch, mode, fs_root, home_dir, follow_symlinks) = + if let Some(ref picker) = *guard { + ( + picker.has_mmap_cache(), + picker.has_content_indexing(), + picker.has_watcher(), + picker.mode(), + picker.fs_root_scanning_enabled(), + picker.home_dir_scanning_enabled(), + picker.follows_symlinks(), + ) + } else { + (false, true, true, FFFMode::default(), false, false, false) + }; + + drop(guard); + + match FilePicker::new_with_shared_state( + inst.picker.clone(), + inst.frecency.clone(), + fff::FilePickerOptions { + base_path: canonical_path.to_string_lossy().to_string(), + enable_mmap_cache: warmup_caches, + enable_content_indexing: content_indexing, + watch, + mode, + cache_budget: None, + follow_symlinks, + enable_fs_root_scanning: fs_root, + enable_home_dir_scanning: home_dir, + }, + ) { + Ok(()) => FffResult::ok_empty(), + Err(e) => FffResult::err(&format!("Failed to init file picker: {}", e)), + } +} + +/// Refresh git status cache. +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_refresh_git_status(fff_handle: *mut c_void) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + match inst.picker.refresh_git_status(&inst.frecency) { + Ok(count) => FffResult::ok_int(count as i64), + Err(e) => FffResult::err(&format!("Failed to refresh git status: {}", e)), + } +} + +/// Track query completion for smart suggestions. +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`. +/// * `query` and `file_path` must be valid null-terminated UTF-8 strings. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_track_query( + fff_handle: *mut c_void, + query: *const c_char, + file_path: *const c_char, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let query_str = match unsafe { cstr_to_str(query) } { + Some(s) => s, + None => return FffResult::err("Query is null or invalid UTF-8"), + }; + + let path_str = match unsafe { cstr_to_str(file_path) } { + Some(s) => s, + None => return FffResult::err("File path is null or invalid UTF-8"), + }; + + let file_path = match fff::path_utils::canonicalize(path_str) { + Ok(p) => p, + Err(e) => return FffResult::err(&format!("Failed to canonicalize path: {}", e)), + }; + + let project_path = { + let guard = match inst.picker.read() { + Ok(g) => g, + Err(_) => return FffResult::ok_int(0), + }; + match guard.as_ref() { + Some(p) => p.base_path().to_path_buf(), + None => return FffResult::ok_int(0), + } + }; + + let mut qt_guard = match inst.query_tracker.write() { + Ok(q) => q, + Err(_) => return FffResult::ok_int(0), + }; + + if let Some(ref mut tracker) = *qt_guard + && let Err(e) = tracker.track_query_completion(query_str, &project_path, &file_path) + { + return FffResult::err(&format!("Failed to track query: {}", e)); + } + + FffResult::ok_int(1) +} + +/// Get historical query by offset (0 = most recent). +/// +/// ## Safety +/// `fff_handle` must be a valid instance pointer from `fff_create_instance`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_get_historical_query( + fff_handle: *mut c_void, + offset: u64, +) -> *mut FffResult { + let inst = match unsafe { instance_ref(fff_handle) } { + Ok(i) => i, + Err(e) => return e, + }; + + let project_path = { + let guard = match inst.picker.read() { + Ok(g) => g, + Err(_) => return FffResult::ok_empty(), + }; + match guard.as_ref() { + Some(p) => p.base_path().to_path_buf(), + None => return FffResult::ok_empty(), + } + }; + + let qt_guard = match inst.query_tracker.read() { + Ok(q) => q, + Err(_) => return FffResult::ok_empty(), + }; + + let tracker = match qt_guard.as_ref() { + Some(t) => t, + None => return FffResult::ok_empty(), + }; + + match tracker.get_historical_query(&project_path, offset as usize) { + Ok(Some(query)) => FffResult::ok_string(&query), + Ok(None) => FffResult::ok_empty(), + Err(e) => FffResult::err(&format!("Failed to get historical query: {}", e)), + } +} + +/// Get health check information. +/// +/// ## Safety +/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`, or null for +/// a limited health check (version + git only). +/// * `test_path` can be null or a valid null-terminated UTF-8 string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_health_check( + fff_handle: *mut c_void, + test_path: *const c_char, +) -> *mut FffResult { + let test_path = unsafe { optional_cstr(test_path) } + .map(PathBuf::from) + .unwrap_or_else(|| std::env::current_dir().unwrap_or_default()); + + let mut health = serde_json::Map::new(); + health.insert( + "version".to_string(), + serde_json::Value::String(env!("CARGO_PKG_VERSION").to_string()), + ); + + // Git info + let mut git_info = serde_json::Map::new(); + let git_version = git2::Version::get(); + let (major, minor, rev) = git_version.libgit2_version(); + git_info.insert( + "libgit2_version".to_string(), + serde_json::Value::String(format!("{}.{}.{}", major, minor, rev)), + ); + + match git2::Repository::discover(&test_path) { + Ok(repo) => { + git_info.insert("available".to_string(), serde_json::Value::Bool(true)); + git_info.insert( + "repository_found".to_string(), + serde_json::Value::Bool(true), + ); + if let Some(workdir) = repo.workdir() { + git_info.insert( + "workdir".to_string(), + serde_json::Value::String(workdir.to_string_lossy().to_string()), + ); + } + } + Err(e) => { + git_info.insert("available".to_string(), serde_json::Value::Bool(true)); + git_info.insert( + "repository_found".to_string(), + serde_json::Value::Bool(false), + ); + git_info.insert( + "error".to_string(), + serde_json::Value::String(e.message().to_string()), + ); + } + } + health.insert("git".to_string(), serde_json::Value::Object(git_info)); + + let inst: Option<&FffInstance> = if fff_handle.is_null() { + None + } else { + Some(unsafe { &*(fff_handle as *const FffInstance) }) + }; + + // File picker info + let mut picker_info = serde_json::Map::new(); + if let Some(inst) = inst { + match inst.picker.read() { + Ok(guard) => { + if let Some(ref picker) = *guard { + picker_info.insert("initialized".to_string(), serde_json::Value::Bool(true)); + picker_info.insert( + "base_path".to_string(), + serde_json::Value::String(picker.base_path().to_string_lossy().to_string()), + ); + picker_info.insert( + "is_scanning".to_string(), + serde_json::Value::Bool(picker.is_scan_active()), + ); + let progress = picker.get_scan_progress(); + picker_info.insert( + "indexed_files".to_string(), + serde_json::Value::Number(progress.scanned_files_count.into()), + ); + } else { + picker_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + } + Err(_) => { + picker_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + picker_info.insert( + "error".to_string(), + serde_json::Value::String("Failed to acquire lock".to_string()), + ); + } + } + } else { + picker_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + health.insert( + "file_picker".to_string(), + serde_json::Value::Object(picker_info), + ); + + // Frecency info + let mut frecency_info = serde_json::Map::new(); + if let Some(inst) = inst { + match inst.frecency.read() { + Ok(guard) => { + frecency_info.insert( + "initialized".to_string(), + serde_json::Value::Bool(guard.is_some()), + ); + if let Some(ref frecency) = *guard + && let Ok(health_data) = frecency.get_health() + { + let mut db_health = serde_json::Map::new(); + db_health.insert( + "path".to_string(), + serde_json::Value::String(health_data.path), + ); + db_health.insert( + "disk_size".to_string(), + serde_json::Value::Number(health_data.disk_size.into()), + ); + frecency_info.insert( + "db_healthcheck".to_string(), + serde_json::Value::Object(db_health), + ); + } + } + Err(_) => { + frecency_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + } + } else { + frecency_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + health.insert( + "frecency".to_string(), + serde_json::Value::Object(frecency_info), + ); + + // Query tracker info + let mut query_info = serde_json::Map::new(); + if let Some(inst) = inst { + match inst.query_tracker.read() { + Ok(guard) => { + query_info.insert( + "initialized".to_string(), + serde_json::Value::Bool(guard.is_some()), + ); + if let Some(ref tracker) = *guard + && let Ok(health_data) = tracker.get_health() + { + let mut db_health = serde_json::Map::new(); + db_health.insert( + "path".to_string(), + serde_json::Value::String(health_data.path), + ); + db_health.insert( + "disk_size".to_string(), + serde_json::Value::Number(health_data.disk_size.into()), + ); + query_info.insert( + "db_healthcheck".to_string(), + serde_json::Value::Object(db_health), + ); + } + } + Err(_) => { + query_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + } + } else { + query_info.insert("initialized".to_string(), serde_json::Value::Bool(false)); + } + health.insert( + "query_tracker".to_string(), + serde_json::Value::Object(query_info), + ); + + match serde_json::to_string(&health) { + Ok(json) => FffResult::ok_string(&json), + Err(e) => FffResult::err(&format!("Failed to serialize health check: {}", e)), + } +} + +/// Free a search result returned by `fff_search`. +/// +/// This frees the `FffSearchResult` struct, its `items` and `scores` arrays, +/// and all heap-allocated strings within each item and score. +/// +/// ## Safety +/// `result` must be a valid pointer previously returned via `FffResult.handle` +/// from `fff_search`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_search_result(result: *mut FffSearchResult) { + if result.is_null() { + return; + } + + unsafe { + let result = Box::from_raw(result); + let count = result.count as usize; + + if !result.items.is_null() { + let mut items = Vec::from_raw_parts(result.items, count, count); + for item in &mut items { + item.free_strings(); + } + } + if !result.scores.is_null() { + let mut scores = Vec::from_raw_parts(result.scores, count, count); + for score in &mut scores { + score.free_strings(); + } + } + } +} + +/// Get a pointer to the `index`-th `FffFileItem` in a search result. +/// +/// Returns null if `result` is null or `index >= result->count`. +/// The returned pointer is valid until the search result is freed. +/// +/// ## Safety +/// `result` must be a valid `FffSearchResult` pointer from `fff_search`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_result_get_item( + result: *const FffSearchResult, + index: u32, +) -> *const FffFileItem { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.items.is_null() { + return std::ptr::null(); + } + unsafe { result.items.add(index as usize) } +} + +/// Get a pointer to the `index`-th `FffScore` in a search result. +/// +/// Returns null if `result` is null or `index >= result->count`. +/// The returned pointer is valid until the search result is freed. +/// +/// ## Safety +/// `result` must be a valid `FffSearchResult` pointer from `fff_search`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_search_result_get_score( + result: *const FffSearchResult, + index: u32, +) -> *const FffScore { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.scores.is_null() { + return std::ptr::null(); + } + unsafe { result.scores.add(index as usize) } +} + +/// Free a grep result returned by `fff_live_grep` or `fff_multi_grep`. +/// +/// This frees the `FffGrepResult` struct, its `items` array, and all +/// heap-allocated strings, match ranges, and context arrays within each match. +/// +/// ## Safety +/// `result` must be a valid pointer previously returned via `FffResult.handle` +/// from `fff_live_grep` or `fff_multi_grep`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_grep_result(result: *mut FffGrepResult) { + if result.is_null() { + return; + } + + unsafe { + let result = Box::from_raw(result); + let count = result.count as usize; + + if !result.items.is_null() { + let mut items = Vec::from_raw_parts(result.items, count, count); + for item in &mut items { + item.free_fields(); + } + } + if !result.regex_fallback_error.is_null() { + drop(CString::from_raw(result.regex_fallback_error)); + } + } +} + +/// Get a pointer to the `index`-th `FffGrepMatch` in a grep result. +/// +/// Returns null if `result` is null or `index >= result->count`. +/// The returned pointer is valid until the grep result is freed. +/// +/// ## Safety +/// `result` must be a valid `FffGrepResult` pointer from `fff_live_grep` or `fff_multi_grep`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_grep_result_get_match( + result: *const FffGrepResult, + index: u32, +) -> *const FffGrepMatch { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.items.is_null() { + return std::ptr::null(); + } + unsafe { result.items.add(index as usize) } +} + +/// Free a scan progress result returned by `fff_get_scan_progress`. +/// +/// ## Safety +/// `result` must be a valid pointer previously returned via `FffResult.handle` +/// from `fff_get_scan_progress`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_scan_progress(result: *mut FffScanProgress) { + if !result.is_null() { + unsafe { drop(Box::from_raw(result)) }; + } +} + +/// Offset a pointer by `byte_offset` bytes. +/// +/// General-purpose utility for FFI consumers that need pointer arithmetic +/// (e.g. iterating over arrays). Returns null if `base` is null. +/// +/// ## Safety +/// The resulting pointer must be within the bounds of the original allocation. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_ptr_offset(base: *const c_void, byte_offset: usize) -> *const c_void { + if base.is_null() { + return std::ptr::null(); + } + unsafe { (base as *const u8).add(byte_offset) as *const c_void } +} + +/// Free a result returned by any `fff_*` function. +/// **IMPORTANT:** this doesn't clean the the internal handle, so it is safe to call right after +/// you handle the error case. +/// +/// Note: Many non-libffi implementations are not supporting struct-by-value returns, so it's more +/// convenient to have pointer returned at most of the time, though allocating result for every call +/// is annoying, so we just rely on the fact that our allocator is good enough. +/// +/// ## Safety +/// `result_ptr` must be a valid pointer returned by a `fff_*` function. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_result(result_ptr: *mut FffResult) { + if result_ptr.is_null() { + return; + } + + unsafe { + let result = Box::from_raw(result_ptr); + if !result.error.is_null() { + drop(CString::from_raw(result.error)); + } + // Note: `handle` is NOT freed here — the caller must free it + // with the appropriate function (fff_destroy, fff_free_search_result, + // fff_free_grep_result, fff_free_string, fff_free_scan_progress, etc.). + } +} + +/// Free a string returned by `fff_*` functions. +/// +/// ## Safety +/// `s` must be a valid C string allocated by this library. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_string(s: *mut c_char) { + unsafe { + if !s.is_null() { + drop(CString::from_raw(s)); + } + } +} + +// --------------------------------------------------------------------------- +// Directory search: free and accessor functions +// --------------------------------------------------------------------------- + +/// Free a directory search result returned by `fff_search_directories`. +/// +/// ## Safety +/// `result` must be a valid pointer previously returned via `FffResult.handle` +/// from `fff_search_directories`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_dir_search_result(result: *mut FffDirSearchResult) { + if result.is_null() { + return; + } + + unsafe { + let result = Box::from_raw(result); + let count = result.count as usize; + + if !result.items.is_null() { + let mut items = Vec::from_raw_parts(result.items, count, count); + for item in &mut items { + item.free_strings(); + } + } + if !result.scores.is_null() { + let mut scores = Vec::from_raw_parts(result.scores, count, count); + for score in &mut scores { + score.free_strings(); + } + } + } +} + +/// Get a pointer to the `index`-th `FffDirItem` in a directory search result. +/// +/// ## Safety +/// `result` must be a valid `FffDirSearchResult` pointer from `fff_search_directories`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_dir_search_result_get_item( + result: *const FffDirSearchResult, + index: u32, +) -> *const FffDirItem { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.items.is_null() { + return std::ptr::null(); + } + unsafe { result.items.add(index as usize) } +} + +/// Get a pointer to the `index`-th `FffScore` in a directory search result. +/// +/// ## Safety +/// `result` must be a valid `FffDirSearchResult` pointer from `fff_search_directories`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_dir_search_result_get_score( + result: *const FffDirSearchResult, + index: u32, +) -> *const FffScore { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.scores.is_null() { + return std::ptr::null(); + } + unsafe { result.scores.add(index as usize) } +} + +// --------------------------------------------------------------------------- +// Mixed search: free and accessor functions +// --------------------------------------------------------------------------- + +/// Free a mixed search result returned by `fff_search_mixed`. +/// +/// ## Safety +/// `result` must be a valid pointer previously returned via `FffResult.handle` +/// from `fff_search_mixed`, or null (no-op). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_free_mixed_search_result(result: *mut FffMixedSearchResult) { + if result.is_null() { + return; + } + + unsafe { + let result = Box::from_raw(result); + let count = result.count as usize; + + if !result.items.is_null() { + let mut items = Vec::from_raw_parts(result.items, count, count); + for item in &mut items { + item.free_strings(); + } + } + if !result.scores.is_null() { + let mut scores = Vec::from_raw_parts(result.scores, count, count); + for score in &mut scores { + score.free_strings(); + } + } + } +} + +/// Get a pointer to the `index`-th `FffMixedItem` in a mixed search result. +/// +/// ## Safety +/// `result` must be a valid `FffMixedSearchResult` pointer from `fff_search_mixed`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_mixed_search_result_get_item( + result: *const FffMixedSearchResult, + index: u32, +) -> *const FffMixedItem { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.items.is_null() { + return std::ptr::null(); + } + unsafe { result.items.add(index as usize) } +} + +/// Get a pointer to the `index`-th `FffScore` in a mixed search result. +/// +/// ## Safety +/// `result` must be a valid `FffMixedSearchResult` pointer from `fff_search_mixed`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fff_mixed_search_result_get_score( + result: *const FffMixedSearchResult, + index: u32, +) -> *const FffScore { + if result.is_null() { + return std::ptr::null(); + } + let result = unsafe { &*result }; + if index >= result.count || result.scores.is_null() { + return std::ptr::null(); + } + unsafe { result.scores.add(index as usize) } +} diff --git a/crates/fff-c/tests/smoke.c b/crates/fff-c/tests/smoke.c new file mode 100644 index 0000000..b021953 --- /dev/null +++ b/crates/fff-c/tests/smoke.c @@ -0,0 +1,89 @@ +/* + * Smoke test for libfff_c — the smallest possible end-to-end exercise of + * the public C API. We: + * + * 1. Create a picker with an `FffCreateOptions` populated via C99 + * designated initializers (the recommended idiom for direct C use). + * 2. Wait for the initial scan to complete. + * 3. Search for "smoke.c". + * 4. Fail unless this very file appears in the results. + * + * Build + run via `make test-c-smoke`. Override $(CC) to test other + * compilers. + */ + +#include +#include +#include + +int main(int argc, char **argv) { + const char *base_path = argc > 1 ? argv[1] : "."; + + // make sure that FFF C api is designed more for FFI rather than for direct C usage (I'm sorry) + struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){ + .version = FFF_CREATE_OPTIONS_VERSION, + .base_path = base_path, + .enable_mmap_cache = false, + .enable_content_indexing = false, + .watch = false, + }); + + if (!create_result->success) { + fprintf(stderr, "fff couldn't create instance: %s\n", + create_result->error ? create_result->error : "?"); + fff_free_result(create_result); + return 1; + } + + void *file_picker = create_result->handle; + fff_free_result(create_result); // safe to drop now: handle outlives the envelope + + struct FffResult *scan_result = fff_wait_for_scan(file_picker, 5000); + if (!scan_result->success) { + fprintf(stderr, "wait_for_scan failed: %s\n", + scan_result->error ? scan_result->error : "?"); + fff_free_result(scan_result); + fff_destroy(file_picker); + return 1; + } + // int_value: 1 = scan completed in time, 0 = timed out. + if (scan_result->int_value == 0) { + fprintf(stderr, "wait_for_scan: timed out before initial scan finished\n"); + fff_free_result(scan_result); + fff_destroy(file_picker); + return 1; + } + fff_free_result(scan_result); + + struct FffResult *res = fff_search(file_picker, "smkoe.c", "", 0, 0, 50, 0, 0); + if (!res->success) { + fprintf(stderr, "search failed: %s\n", res->error ? res->error : "?"); + fff_free_result(res); + fff_destroy(file_picker); + return 1; + } + + struct FffSearchResult *sr = (struct FffSearchResult *)res->handle; + uint32_t total = sr->count; + int found = 0; + for (uint32_t i = 0; i < sr->count; i++) { + const char *path = sr->items[i].relative_path; + if (path && strstr(path, "smoke.c")) { + found = 1; + fprintf(stderr, "found self: %s\n", path); + break; + } + } + + fff_free_search_result(sr); + fff_free_result(res); + fff_destroy(file_picker); + + if (!found) { + fprintf(stderr, "FAIL: smoke.c not in search results (count=%u)\n", total); + return 1; + } + + fprintf(stderr, "PASS\n"); + return 0; +} diff --git a/crates/fff-core/Cargo.toml b/crates/fff-core/Cargo.toml new file mode 100644 index 0000000..5600fd3 --- /dev/null +++ b/crates/fff-core/Cargo.toml @@ -0,0 +1,95 @@ +[package] +name = "fff-search" +version = "0.9.6" +edition = "2024" +license = "MIT" +authors = ["Dmitriy Kovalenko "] +description = "Faboulous & Fast File Finder - a fast and extremely correct file finder SDK with typo resistance, SIMD, prefiltering, and more" + +[lib] +path = "src/lib.rs" +crate-type = ["rlib", "staticlib", "cdylib"] + +[[bench]] +name = "parse_bench" +harness = false + +[[bench]] +name = "bigram_bench" +harness = false + +[[bench]] +name = "memmem_bench" +harness = false + +[[bench]] +name = "glob_bench" +harness = false +required-features = ["zlob"] + +[features] +# `ripgrep` is the pure-Rust walker/glob backend and is on by default so +# consumers build without a Zig toolchain. CI/release opt into zlob via +# `--no-default-features --features zlob`. +default = ["ripgrep"] +# Enable C FFI exports +ffi = [] +# Enables POC definition classification for grep result matched lines +definitions = [] +# Pure-Rust filesystem walker + glob matcher (ignore + globset crates). +ripgrep = ["dep:ignore", "dep:globset", "fff-query-parser/ripgrep"] +# Call mi_collect(true) after large allocator churn (bigram build). +# Requires mimalloc to be the global allocator (linked by fff-nvim). +mimalloc-collect = ["dep:libmimalloc-sys"] +# Use zlob (Zig-compiled C globbing library) for glob matching. +# Requires Zig to be installed. When disabled, falls back to globset (pure Rust). +zlob = ["dep:zlob", "fff-query-parser/zlob"] + +[dependencies] +ahash = { workspace = true } +rayon = { workspace = true } +smallvec = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +fff-query-parser = { workspace = true , version = "0.9.0" } +blake3 = { workspace = true } +dirs = { workspace = true } +libc = "0.2" +git2 = { workspace = true } +glidesort = { workspace = true } +globset = { workspace = true, optional = true } +fff-grep = { workspace = true , version = "0.9.0" } +aho-corasick = "1" +memchr = "2" +heed = { workspace = true } +ignore = { workspace = true, optional = true } +memmap2 = { workspace = true } +neo_frizbee = { workspace = true } +notify = { workspace = true } +notify-debouncer-full = { workspace = true } +parking_lot = { workspace = true } +pathdiff = { workspace = true } +regex = { workspace = true } +regex-syntax = "0.8" +serde = { version = "1.0", features = ["derive"] } +tracing-appender = "0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +zlob = { workspace = true, optional = true } +libmimalloc-sys = { version = "0.1", optional = true, features = ["extended", "local_dynamic_tls"] } +mimalloc = { version = "0.1", optional = true, features = ["local_dynamic_tls"] } +# Platform-specific: dunce for Windows to avoid \\?\ extended path prefix +[target.'cfg(windows)'.dependencies] +dunce = { workspace = true } + +# signal-hook only compiles on unix; we wrap the SIGSEGV handler behind cfg(unix) +[target.'cfg(unix)'.dependencies] +signal-hook-registry = { workspace = true } + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +ctor = "0.2" +proptest = { version = "1", default-features = false, features = ["std", "fork"] } +rand = { version = "0.8", features = ["small_rng"] } +tempfile = "3.8" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + diff --git a/crates/fff-core/README.md b/crates/fff-core/README.md new file mode 100644 index 0000000..e7ee90e --- /dev/null +++ b/crates/fff-core/README.md @@ -0,0 +1,29 @@ +# fff + +fff is a file search toolkit. It is faster than ripgrep and fzf and designed for a long running applications like file editors, ai agents, or file exploerers. + +> [!Important performance information] +> For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine. + +## Features + +- Fuzzy file name search +- Typo resistance +- Frecency and query history ranking +- Native git support via libgit +- Advanced ranking +- Grep functionality with SIMD optimized plain matcher and regex +- Multi grep using aho-corasick algorithm +- Efficient memory mapping for file system +- Cross platform support (Linux, Windows, MacOS) +- Advnaced constraints syntax allowing to prefilter based on git status, glob, extension, size, timing and more + +## Performance + +FFF is designed for high performance and low latency. SIMD optimized where needed, parallelized for multi core systems, efficient sorting and ranking algorithms, memaps and much more. + +On MacOS FFF is about 20-50 times faster than ripgrep for content search and around 10 times faster than fzf for file name search. + +## Documentation + +Refer rust docs https://docs.rs/crate/fff-search/latest diff --git a/crates/fff-core/benches/bigram_bench.rs b/crates/fff-core/benches/bigram_bench.rs new file mode 100644 index 0000000..85c7466 --- /dev/null +++ b/crates/fff-core/benches/bigram_bench.rs @@ -0,0 +1,164 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use fff_search::bigram_filter::{BigramFilter, BigramIndexBuilder}; + +/// Build a realistic bigram index for benchmarking. +/// Simulates a large repo by generating varied content per file. +fn build_test_index(file_count: usize) -> BigramFilter { + let builder = BigramIndexBuilder::new(file_count); + let skip_builder = BigramIndexBuilder::new(file_count); + + for i in 0..file_count { + // Generate varied content so we get a mix of sparse and dense columns + let content = format!( + "struct File{i} {{ fn process() {{ let controller = read(path); }} }} // module {i}" + ); + builder.add_file_content(&skip_builder, i, content.as_bytes()); + } + + let mut index = builder.compress(None); + let skip_index = skip_builder.compress(Some(12)); + index.set_skip_index(skip_index); + index +} + +fn bench_bigram_query(c: &mut Criterion) { + let file_counts = [10_000, 100_000, 500_000]; + + for &file_count in &file_counts { + let index = build_test_index(file_count); + eprintln!( + "Index ({} files): {} columns", + file_count, + index.columns_used(), + ); + + let mut group = c.benchmark_group(format!("bigram_query_{file_count}")); + group.sample_size(500); + + let queries: &[(&str, &[u8])] = &[ + ("short_2char", b"st"), + ("medium_6char", b"struct"), + ("long_14char", b"let controller"), + ("multi_word", b"fn process"), + ]; + + for (name, query) in queries { + group.bench_with_input(BenchmarkId::from_parameter(name), query, |b, q| { + b.iter(|| { + let result = index.query(black_box(q)); + black_box(&result); + }); + }); + } + + group.finish(); + } +} + +fn bench_bigram_is_candidate(c: &mut Criterion) { + let index = build_test_index(500_000); + let candidates = match index.query(b"struct") { + Some(c) => c, + None => { + // All bigrams ubiquitous at this size — skip candidate benches + eprintln!("Skipping is_candidate bench: query returned None (all bigrams ubiquitous)"); + return; + } + }; + + c.bench_function("is_candidate_500k", |b| { + b.iter(|| { + let mut count = 0u32; + for i in 0..500_000 { + if BigramFilter::is_candidate(black_box(&candidates), i) { + count += 1; + } + } + black_box(count) + }); + }); + + c.bench_function("count_candidates_500k", |b| { + b.iter(|| BigramFilter::count_candidates(black_box(&candidates))); + }); +} + +fn bench_bigram_build(c: &mut Criterion) { + let mut group = c.benchmark_group("bigram_build"); + group.sample_size(10); + + let file_counts = [10_000, 100_000]; + + for &file_count in &file_counts { + // Pre-generate content so we only measure index building. + // Short content (~85 bytes/file) exercises the scalar fast path. + let contents: Vec = (0..file_count) + .map(|i| { + format!( + "struct File{i} {{ fn process() {{ let controller = read(path); }} }} // mod {i}" + ) + }) + .collect(); + + group.bench_with_input( + BenchmarkId::new("short_content", file_count), + &file_count, + |b, &fc| { + b.iter(|| { + let builder = BigramIndexBuilder::new(fc); + let skip_builder = BigramIndexBuilder::new(fc); + for (i, content) in contents.iter().enumerate() { + builder.add_file_content(&skip_builder, i, content.as_bytes()); + } + let index = builder.compress(None); + black_box(index.columns_used()) + }); + }, + ); + + // Long content (~4 KB/file) exercises the SIMD pre-pass path. + // Build a realistic-looking source-like blob by repeating snippets. + let long_contents: Vec = (0..file_count) + .map(|i| { + let mut s = String::with_capacity(4096); + for j in 0..50 { + s.push_str(&format!( + "pub fn handler_{i}_{j}(ctx: &Context) -> Result {{\n" + )); + s.push_str(" let parsed = ctx.parse()?;\n"); + s.push_str(" let validated = parsed.validate()?;\n"); + s.push_str(&format!(" ctx.respond(validated, {}).await\n", j)); + s.push_str("}\n\n"); + } + s + }) + .collect(); + + group.bench_with_input( + BenchmarkId::new("long_content", file_count), + &file_count, + |b, &fc| { + b.iter(|| { + let builder = BigramIndexBuilder::new(fc); + let skip_builder = BigramIndexBuilder::new(fc); + for (i, content) in long_contents.iter().enumerate() { + builder.add_file_content(&skip_builder, i, content.as_bytes()); + } + let index = builder.compress(None); + black_box(index.columns_used()) + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_bigram_query, + bench_bigram_is_candidate, + bench_bigram_build, +); + +criterion_main!(benches); diff --git a/crates/fff-core/benches/glob_bench.rs b/crates/fff-core/benches/glob_bench.rs new file mode 100644 index 0000000..914fa0c --- /dev/null +++ b/crates/fff-core/benches/glob_bench.rs @@ -0,0 +1,386 @@ +//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs: +//! +//! 1. Current: `zlob_match_paths` -> collect `as_ptr()` into AHashSet, filter paths +//! by pointer to recover indices. +//! 2. Free fn: `zlob_match_paths_indices` (added in zlob 1.4) — indices direct from C. +//! 3. Compiled: `ZlobPattern::compile` + `match_indices` — same indices path, but with +//! a precompiled pattern (reusable). For one-shot it should match (2); the win +//! appears if the pattern is reused (chunked / repeated calls). +use ahash::AHashSet; +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use zlob::{ZlobFlags, ZlobPattern, zlob_match_paths, zlob_match_paths_indices}; + +fn make_paths(n: usize) -> Vec { + let exts = ["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"]; + let dirs = [ + "src/core", + "src/ui", + "crates/fff-core/src", + "lua/fff", + "tests/integration", + "vendor/lib", + "node_modules/foo/bar", + "docs/internal", + ]; + let mut out = Vec::with_capacity(n); + for i in 0..n { + let dir = dirs[i % dirs.len()]; + let ext = exts[i % exts.len()]; + out.push(format!("{dir}/file_{i}.{ext}")); + } + out +} + +fn current_impl(pattern: &str, paths: &[&str]) -> AHashSet { + let Ok(Some(matches)) = zlob_match_paths(pattern, paths, ZlobFlags::RECOMMENDED) else { + return AHashSet::new(); + }; + let matched_set: AHashSet = matches.iter().map(|s| s.as_ptr() as usize).collect(); + paths + .iter() + .enumerate() + .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize))) + .map(|(i, _)| i) + .collect() +} + +fn indices_free_fn(pattern: &str, paths: &[&str]) -> AHashSet { + let Ok(hits) = zlob_match_paths_indices(pattern, paths, ZlobFlags::RECOMMENDED) else { + return AHashSet::new(); + }; + hits.to_iter().collect() +} + +fn compiled_pattern(pattern: &str, paths: &[&str]) -> AHashSet { + let Ok(p) = ZlobPattern::compile(pattern, ZlobFlags::RECOMMENDED) else { + return AHashSet::new(); + }; + let Ok(hits) = p.match_indices(paths, ZlobFlags::RECOMMENDED) else { + return AHashSet::new(); + }; + hits.to_iter().collect() +} + +fn bench_glob_strategies(c: &mut Criterion) { + let path_counts = [1_000usize, 10_000, 100_000]; + let patterns: &[(&str, &str)] = &[ + ("ext_rs", "**/*.rs"), + ("dir_glob", "src/**/*.{ts,lua}"), + ("literal_seg", "**/node_modules/**"), + ("brace_multi", "**/*.{rs,ts,lua,md}"), + ]; + + for &count in &path_counts { + let owned = make_paths(count); + let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect(); + + let mut group = c.benchmark_group(format!("glob_{count}")); + group.sample_size(50); + + for &(name, pat) in patterns { + let id_curr = BenchmarkId::new("current_ptr_trick", name); + group.bench_with_input(id_curr, &pat, |b, &pat| { + b.iter(|| { + let r = current_impl(black_box(pat), black_box(&paths)); + black_box(r); + }); + }); + + let id_idx = BenchmarkId::new("match_indices_fn", name); + group.bench_with_input(id_idx, &pat, |b, &pat| { + b.iter(|| { + let r = indices_free_fn(black_box(pat), black_box(&paths)); + black_box(r); + }); + }); + + let id_comp = BenchmarkId::new("compiled_pattern", name); + group.bench_with_input(id_comp, &pat, |b, &pat| { + b.iter(|| { + let r = compiled_pattern(black_box(pat), black_box(&paths)); + black_box(r); + }); + }); + } + + group.finish(); + } +} + +/// Hot-loop: pattern compiled ONCE, matched many times against fresh path slices. +/// Models a hypothetical change where we cache compiled patterns across calls. +fn bench_compiled_reuse(c: &mut Criterion) { + let owned = make_paths(10_000); + let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect(); + let pat = "**/*.{rs,ts,lua,md}"; + + let mut group = c.benchmark_group("glob_reuse_10k"); + group.sample_size(100); + + group.bench_function("recompile_each_time", |b| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap(); + let hits = p + .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED) + .unwrap(); + black_box(hits.len()); + }); + }); + + let compiled = ZlobPattern::compile(pat, ZlobFlags::RECOMMENDED).unwrap(); + group.bench_function("reuse_compiled", |b| { + b.iter(|| { + let hits = compiled + .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED) + .unwrap(); + black_box(hits.len()); + }); + }); + + group.finish(); +} + +/// End-to-end: build the lookup AND iterate items checking membership, modeling the +/// real call shape in `apply_constraints` (filter loop reads the result for every item). +fn bench_full_pipeline(c: &mut Criterion) { + bench_full_pipeline_size(c, 100_000); + bench_full_pipeline_size(c, 500_000); +} + +fn bench_full_pipeline_size(c: &mut Criterion, count: usize) { + let owned = make_paths(count); + let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect(); + let pat = "**/*.{rs,ts,lua,md}"; + + let mut group = c.benchmark_group(format!("glob_full_pipeline_{count}")); + group.sample_size(50); + + // (A) current: indices -> AHashSet -> per-item set.contains + group.bench_function("indices_to_ahashset_then_filter", |b| { + b.iter(|| { + let hits = + zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap(); + let set: AHashSet = hits.to_iter().collect(); + let count = (0..paths.len()).filter(|i| set.contains(i)).count(); + black_box(count); + }); + }); + + // (B) indices -> Vec bitmap -> per-item array lookup + group.bench_function("indices_to_bitmap_then_filter", |b| { + b.iter(|| { + let hits = + zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap(); + let mut mask = vec![false; paths.len()]; + for i in hits.to_iter() { + mask[i] = true; + } + let count = (0..paths.len()).filter(|&i| mask[i]).count(); + black_box(count); + }); + }); + + // (C) compiled pattern + per-item matches() inside the filter loop. No batch. + group.bench_function("compiled_per_item_matches", |b| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap(); + let count = paths.iter().filter(|path| p.matches_default(path)).count(); + black_box(count); + }); + }); + + // (D) compiled pattern + chunked batch -> Vec bitmap. Best of both: + // SIMD batch wins inside chunks, no global allocation pressure, O(1) lookup. + group.bench_function("compiled_chunked_to_bitmap", |b| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap(); + let mut mask = vec![false; paths.len()]; + for (chunk_idx, chunk) in paths.chunks(512).enumerate() { + let base = chunk_idx * 512; + let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap(); + for i in hits.to_iter() { + mask[base + i] = true; + } + } + let count = (0..paths.len()).filter(|&i| mask[i]).count(); + black_box(count); + }); + }); + + // (E') indices -> bit-packed Vec -> per-item bit test + group.bench_function("indices_to_bitset_then_filter", |b| { + b.iter(|| { + let hits = + zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap(); + let words = paths.len().div_ceil(64); + let mut bits = vec![0u64; words]; + for i in hits.to_iter() { + bits[i >> 6] |= 1u64 << (i & 63); + } + let count = (0..paths.len()) + .filter(|&i| (bits[i >> 6] >> (i & 63)) & 1 == 1) + .count(); + black_box(count); + }); + }); + + // (E) (D) but larger chunk + group.bench_function("compiled_chunked_4096_to_bitmap", |b| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap(); + let mut mask = vec![false; paths.len()]; + for (chunk_idx, chunk) in paths.chunks(4096).enumerate() { + let base = chunk_idx * 4096; + let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap(); + for i in hits.to_iter() { + mask[base + i] = true; + } + } + let count = (0..paths.len()).filter(|&i| mask[i]).count(); + black_box(count); + }); + }); + + group.finish(); +} + +/// Mixed-constraint pipeline: glob + ext. Compare pre-pass batch (current) vs +/// inline `ZlobPattern::matches` after the cheap ext check rejects items. +/// +/// Variables: ext rejection rate. Extreme cases reveal where each strategy wins. +fn bench_mixed_pipeline(c: &mut Criterion) { + let count = 100_000; + let owned = make_paths(count); + let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect(); + let glob_pat = "**/*.{rs,ts,lua,md}"; + + // 4 ext sets: from very selective (1/10 paths kept) to permissive (kept all). + let scenarios: &[(&str, &[&str])] = &[ + ("ext_1of10", &["rs"]), + ("ext_4of10", &["rs", "ts", "lua", "md"]), + ( + "ext_8of10", + &["rs", "ts", "lua", "md", "toml", "go", "py", "c"], + ), + ( + "ext_all", + &["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"], + ), + ]; + + fn ext_match(name: &str, exts: &[&str]) -> bool { + exts.iter().any(|e| { + let bytes = name.as_bytes(); + let elen = e.len(); + bytes.len() > elen + 1 + && bytes[bytes.len() - elen - 1] == b'.' + && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes()) + }) + } + + let mut group = c.benchmark_group("glob_mixed_100k"); + group.sample_size(50); + + for &(name, exts) in scenarios { + // (A) PRE-PASS: build bitmap for ALL paths, then per-item ext-then-bitmap. + let id_pre = BenchmarkId::new("prepass_bitmap", name); + group.bench_with_input(id_pre, &exts, |b, &exts| { + b.iter(|| { + let hits = + zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED) + .unwrap(); + let mut mask = vec![false; paths.len()]; + for i in hits.to_iter() { + mask[i] = true; + } + let count = paths + .iter() + .enumerate() + .filter(|&(_, p)| ext_match(p, exts)) + .filter(|&(i, _)| mask[i]) + .count(); + black_box(count); + }); + }); + + // (B) INLINE: compile once, per-item ext check first, then matches() only on survivors. + let id_inline = BenchmarkId::new("inline_compiled", name); + group.bench_with_input(id_inline, &exts, |b, &exts| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap(); + let count = paths + .iter() + .filter(|path| ext_match(path, exts)) + .filter(|path| p.matches_default(path)) + .count(); + black_box(count); + }); + }); + } + + group.finish(); +} + +/// Compare hand-rolled `file_has_extension` byte compare vs compiling extensions +/// into a single brace glob `**/*.{rs,ts,lua,md}` and dispatching through zlob. +/// Both share the same per-item "filter then count" shape. +fn bench_extensions_vs_glob(c: &mut Criterion) { + let owned = make_paths(100_000); + let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect(); + let exts = ["rs", "ts", "lua", "md"]; + let glob_pat = "**/*.{rs,ts,lua,md}"; + + fn ext_match(name: &str, exts: &[&str]) -> bool { + let bytes = name.as_bytes(); + exts.iter().any(|e| { + let elen = e.len(); + bytes.len() > elen + 1 + && bytes[bytes.len() - elen - 1] == b'.' + && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes()) + }) + } + + let mut group = c.benchmark_group("ext_vs_glob_100k"); + group.sample_size(50); + + group.bench_function("file_has_extension_loop", |b| { + b.iter(|| { + let count = paths.iter().filter(|p| ext_match(p, &exts)).count(); + black_box(count); + }); + }); + + group.bench_function("compiled_brace_glob_inline", |b| { + b.iter(|| { + let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap(); + let count = paths.iter().filter(|path| p.matches_default(path)).count(); + black_box(count); + }); + }); + + group.bench_function("brace_glob_prepass_bitmap", |b| { + b.iter(|| { + let hits = + zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED) + .unwrap(); + let mut mask = vec![false; paths.len()]; + for i in hits.to_iter() { + mask[i] = true; + } + let count = (0..paths.len()).filter(|&i| mask[i]).count(); + black_box(count); + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + bench_glob_strategies, + bench_compiled_reuse, + bench_full_pipeline, + bench_mixed_pipeline, + bench_extensions_vs_glob +); +criterion_main!(benches); diff --git a/crates/fff-core/benches/memmem_bench.rs b/crates/fff-core/benches/memmem_bench.rs new file mode 100644 index 0000000..cacc786 --- /dev/null +++ b/crates/fff-core/benches/memmem_bench.rs @@ -0,0 +1,85 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use fff_search::simd_string_utils::memmem; +use std::path::Path; + +/// Load real source files from the repository as benchmark haystacks. +/// Falls back to concatenating all .rs files under crates/ if specific files are missing. +fn load_real_files() -> Vec<(&'static str, Vec)> { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); // crates/fff-core + let repo_root = Path::new(manifest_dir).parent().unwrap().parent().unwrap(); + + let files: &[(&str, &str)] = &[ + ("grep.rs/80KB", "crates/fff-core/src/grep.rs"), + ("file_picker.rs/53KB", "crates/fff-core/src/file_picker.rs"), + ("picker_ui.lua/96KB", "lua/fff/picker_ui.lua"), + ]; + + let mut result = Vec::new(); + for &(label, rel_path) in files { + let full_path = repo_root.join(rel_path); + if let Ok(data) = std::fs::read(&full_path) { + result.push((label, data)); + } + } + + // Also create a large synthetic file by concatenating all three + if result.len() == 3 { + let mut combined = Vec::new(); + for (_, data) in &result { + combined.extend_from_slice(data); + } + // Repeat to get ~1MB + let base = combined.clone(); + while combined.len() < 1024 * 1024 { + combined.extend_from_slice(&base); + } + combined.truncate(1024 * 1024); + result.push(("combined/1MB", combined)); + } + + result +} + +fn bench_memmem(c: &mut Criterion) { + let mut group = c.benchmark_group("simd_string_utils_memmem"); + + let files = load_real_files(); + assert!(!files.is_empty(), "No source files found for benchmarking"); + + // Needles chosen to exercise different false-positive rates: + // + // "hit" needles: strings that actually appear in these source files. + // "miss" needles: strings with common first-bytes (lots of false positives + // for memchr2) but that don't exist in any of the files. + let needles: &[(&str, &[u8])] = &[ + // Hits — real identifiers from the codebase + ("short/hit/fn", b"fn"), + ("short/hit/self", b"self"), + ("medium/hit", b"search_file"), + ("long/hit", b"content_cache_budget"), + // Misses — common first-bytes, guaranteed not in source + ("short/miss", b"zqxjv"), + ("medium/miss", b"fluxcapacitor"), + ("long/miss", b"quantum_entanglement_resolver"), + ]; + + for (file_label, haystack) in &files { + for &(needle_label, needle) in needles { + let needle_lower: Vec = needle.iter().map(|b| b.to_ascii_lowercase()).collect(); + let id = format!("{file_label}/{needle_label}"); + + group.bench_with_input( + BenchmarkId::new("find", &id), + &(haystack, &needle_lower), + |b, &(h, n)| { + b.iter(|| black_box(memmem::find(h, n))); + }, + ); + } + } + + group.finish(); +} + +criterion_group!(benches, bench_memmem); +criterion_main!(benches); diff --git a/crates/fff-core/benches/parse_bench.rs b/crates/fff-core/benches/parse_bench.rs new file mode 100644 index 0000000..cd6e8d2 --- /dev/null +++ b/crates/fff-core/benches/parse_bench.rs @@ -0,0 +1,180 @@ +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use fff_query_parser::*; + +fn bench_parse_simple(c: &mut Criterion) { + let parser = QueryParser::default(); + + c.bench_function("parse_simple_text", |b| { + b.iter(|| parser.parse(black_box("hello world"))); + }); + + c.bench_function("parse_extension", |b| { + b.iter(|| parser.parse(black_box("*.rs"))); + }); + + c.bench_function("parse_text_with_extension", |b| { + b.iter(|| parser.parse(black_box("name *.rs"))); + }); +} + +fn bench_parse_complex(c: &mut Criterion) { + let parser = QueryParser::default(); + + c.bench_function("parse_complex_mixed", |b| { + b.iter(|| parser.parse(black_box("src name *.rs !test /lib/ status:modified"))); + }); + + c.bench_function("parse_glob", |b| { + b.iter(|| parser.parse(black_box("**/*.rs"))); + }); + + c.bench_function("parse_multiple_constraints", |b| { + b.iter(|| parser.parse(black_box("*.rs *.toml *.md !test !node_modules /src/"))); + }); +} + +fn bench_parse_realistic_queries(c: &mut Criterion) { + let parser = QueryParser::default(); + + let queries = vec![ + "file", + "test", + "mod.rs", + "src/*.rs", + "lib test", + "*.rs !test", + "src/lib/*.rs", + "/src/ name", + "status:modified *.rs", + "type:rust test !node_modules", + ]; + + let mut group = c.benchmark_group("realistic_queries"); + for query in queries.iter() { + group.throughput(Throughput::Bytes(query.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(query), query, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + } + group.finish(); +} + +fn bench_parse_various_lengths(c: &mut Criterion) { + let parser = QueryParser::default(); + + let short = "*.rs"; + let medium = "src name *.rs !test"; + let long = "src lib test name *.rs *.toml !node_modules !test /src/ /lib/ status:modified"; + let very_long = + "a b c d e f g h i j k l m n o p q r s t u v w x y z *.rs *.toml *.md *.txt *.js"; + + let mut group = c.benchmark_group("query_lengths"); + + group.throughput(Throughput::Bytes(short.len() as u64)); + group.bench_with_input(BenchmarkId::new("short", short.len()), &short, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(medium.len() as u64)); + group.bench_with_input(BenchmarkId::new("medium", medium.len()), &medium, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(long.len() as u64)); + group.bench_with_input(BenchmarkId::new("long", long.len()), &long, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(very_long.len() as u64)); + group.bench_with_input( + BenchmarkId::new("very_long", very_long.len()), + &very_long, + |b, q| { + b.iter(|| parser.parse(black_box(q))); + }, + ); + + group.finish(); +} + +fn bench_config_comparison(c: &mut Criterion) { + let file_picker = QueryParser::new(FileSearchConfig); + let grep = QueryParser::new(GrepConfig); + + let query = "src name *.rs !test"; + + let mut group = c.benchmark_group("config_comparison"); + + group.bench_function("file_picker_config", |b| { + b.iter(|| file_picker.parse(black_box(query))); + }); + + group.bench_function("grep_config", |b| { + b.iter(|| grep.parse(black_box(query))); + }); + + group.finish(); +} + +fn bench_constraint_types(c: &mut Criterion) { + let parser = QueryParser::default(); + + let mut group = c.benchmark_group("constraint_types"); + + group.bench_function("extension", |b| { + b.iter(|| parser.parse(black_box("*.rs"))); + }); + + group.bench_function("glob", |b| { + b.iter(|| parser.parse(black_box("**/*.rs"))); + }); + + group.bench_function("exclude", |b| { + b.iter(|| parser.parse(black_box("!test"))); + }); + + group.bench_function("path_segment", |b| { + b.iter(|| parser.parse(black_box("/src/"))); + }); + + group.bench_function("git_status", |b| { + b.iter(|| parser.parse(black_box("status:modified"))); + }); + + group.bench_function("file_type", |b| { + b.iter(|| parser.parse(black_box("type:rust"))); + }); + + group.finish(); +} + +fn bench_worst_case(c: &mut Criterion) { + let parser = QueryParser::default(); + + // Worst case: many constraints that all need to be checked + let worst_case = "a b c d e f g h i j k l m n o p q r s t u v w x y z"; + + c.bench_function("worst_case_many_text_tokens", |b| { + b.iter(|| parser.parse(black_box(worst_case))); + }); + + // Many constraints + let many_constraints = "*.rs *.toml *.md *.txt *.js *.ts *.jsx *.tsx *.vue *.svelte"; + + c.bench_function("worst_case_many_constraints", |b| { + b.iter(|| parser.parse(black_box(many_constraints))); + }); +} + +criterion_group!( + benches, + bench_parse_simple, + bench_parse_complex, + bench_parse_realistic_queries, + bench_parse_various_lengths, + bench_config_comparison, + bench_constraint_types, + bench_worst_case, +); + +criterion_main!(benches); diff --git a/crates/fff-core/build.rs b/crates/fff-core/build.rs new file mode 100644 index 0000000..1e4dfaa --- /dev/null +++ b/crates/fff-core/build.rs @@ -0,0 +1,45 @@ +fn main() { + // Opt-in cfg for the long-running randomized stress tests + // used by tests/fuzz_git_watcher_stress.rs + println!("cargo::rustc-check-cfg=cfg(stress)"); + + // When the `zlob` feature is enabled (Zig-compiled C library): + // On Windows MSVC, explicitly link the C runtime libraries. + // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the + // MSVC CRT, so symbols like strcmp, memcpy etc. would be unresolved. + if std::env::var("CARGO_FEATURE_ZLOB").is_ok() { + if !zig_available() { + panic!( + "The `zlob` feature is enabled but Zig is not installed. \ + Install Zig (https://ziglang.org/download/) or build without \ + `--features zlob`." + ); + } + + let target = std::env::var("TARGET").unwrap_or_default(); + if target.contains("windows") && target.contains("msvc") { + println!("cargo:rustc-link-lib=msvcrt"); + println!("cargo:rustc-link-lib=ucrt"); + println!("cargo:rustc-link-lib=vcruntime"); + } + } else if std::env::var("CARGO_PRIMARY_PACKAGE").is_ok() && zig_available() { + // Hint: if Zig is available but the zlob feature wasn't enabled, + // let the developer know they can get faster glob matching. + // Only emit this hint when this crate is the primary package to + // avoid noisy warnings for downstream consumers. + println!( + "cargo:warning=Zig detected but `zlob` feature is not enabled. \ + Build with `--features zlob` for faster glob matching." + ); + } +} + +/// Probe the system for a working Zig installation. +fn zig_available() -> bool { + std::process::Command::new("zig") + .arg("version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok() +} diff --git a/crates/fff-core/src/background_watcher.rs b/crates/fff-core/src/background_watcher.rs new file mode 100644 index 0000000..72e1b9f --- /dev/null +++ b/crates/fff-core/src/background_watcher.rs @@ -0,0 +1,853 @@ +use crate::constants::MAX_OVERFLOW_FILES; +use crate::error::Error; +use crate::file_picker::FFFMode; +use crate::git_status_worker::GitStatusWorker; +use crate::shared::{SharedFilePicker, SharedFrecency}; +use crate::sort_buffer::sort_with_buffer; +use git2::Repository; +use notify::event::{AccessKind, AccessMode}; +use notify::{Config, EventKind, EventKindMask, RecursiveMode}; +use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, NoCache, new_debouncer_opt}; +use parking_lot::Mutex; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::mpsc; +use std::time::Duration; +use tracing::{Level, debug, error, info, warn}; + +type Debouncer = notify_debouncer_full::Debouncer; + +/// Owns the file-system watcher and guarantees that all background threads +/// are fully joined before `stop()` / `Drop` returns. +pub struct BackgroundWatcher { + debouncer: Arc>>, + watch_tx: Option>, + owner_thread: Option>, +} + +const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(50); +/// On macOS, each `watch()` call creates a separate FSEventStream. When the +/// number of directories exceeds this threshold we fall back to a single +/// recursive watch to avoid exhausting the per-process stream limit. +const MAX_MACOS_NONRECURSIVE_WATCHES: usize = 4096; +/// Minimum seconds between frecency tracks of the same file in AI mode. +/// Prevents score inflation from rapid burst edits by AI agents. +const AI_MODE_COOLDOWN_SECS: u64 = 5 * 60; + +impl BackgroundWatcher { + #[allow(clippy::too_many_arguments)] + pub fn new( + base_path: PathBuf, + git_workdir: Option, + shared_picker: SharedFilePicker, + shared_frecency: SharedFrecency, + mode: FFFMode, + enable_fs_root_scanning: bool, + enable_home_dir_scanning: bool, + git_status_worker: Arc, + trace_span: tracing::Span, + ) -> Result { + info!( + "Initializing background watcher for path: {}, mode: {:?}", + base_path.display(), + mode, + ); + + // by default we do not want to allow users to search their FS root, this is very error prone + // though some consumers would specifically allow that e.g. unikernels, windows disc + // partition or sub file systems. By default - fail, unless user permits + let is_fs_root = base_path.parent().is_none(); + // use rust's path api for maximum reliability of the comparison + let is_home_dir = Some(&base_path) == dirs::home_dir().as_ref(); + + if (is_fs_root && !enable_fs_root_scanning) || (is_home_dir && !enable_home_dir_scanning) { + return Err(Error::FilesystemRoot(base_path)); + } + + // macOS: always use a single recursive FSEvent stream. + // Per-dir NonRecursive watches create one FSEvent stream per dir. + // The per-process FSEvent cap is lower than expected in practice + // (4096 per process, but FFF usually is running within code editors), + // and each failed `watch()` after the cap blocks ~40 ms on kernel retry. + // Yes we pay for filtering events on handler phase but it is usable + // + // Windows doesn't seem to have a hard cap, but in practice non recursive watching + // does a way worse job and often looses events which is not an option for us. + // + // Linux keeps the per-dir NonRecursive strategy: inotify has no + // kernel-level watcher recursion, so we have to manually watch every single interested + // directory for watch events which is in practice stable and fast if system has enough + // spare watcher (configurable by the user, usually 100k - 1m) + let use_recursive = cfg!(any(target_os = "macos", target_os = "windows")); + + let (watch_tx, watch_rx) = mpsc::channel::(); + let watch_tx_for_debouncer = watch_tx.clone(); + + let owner_weak_picker = shared_picker.weaken(); + let owner_git_workdir = git_workdir.clone(); + let owner_git_worker = Arc::clone(&git_status_worker); + + let debouncer = Self::create_debouncer( + base_path, + git_workdir, + shared_picker, + shared_frecency, + mode, + use_recursive, + watch_tx_for_debouncer, + git_status_worker, + )?; + + info!("Background file watcher initialized successfully"); + + // debouncer is shared with the owner thread, once it's dropped the thread is closed + let debouncer = Arc::new(Mutex::new(Some(debouncer))); + // Only the Linux per-dir-watch branch needs this clone; on other + // platforms the owner thread never touches the debouncer. + #[cfg(target_os = "linux")] + let owner_debouncer = Arc::clone(&debouncer); + + let owner_span = trace_span.clone(); + let owner_thread = std::thread::Builder::new() + .name("fff-watcher-own".into()) + .spawn(move || { + let _g = owner_span.enter(); + while let Ok(dir) = watch_rx.recv() { + // if the picker is dropped we do need to exit the loop + let Some(strong_picker) = owner_weak_picker.upgrade() else { + break; + }; + + // Only inotify (Linux) has no kernel-level recursion, so + // it's the only platform that needs a per-subdir watch to + // be registered at runtime. macOS FSEvents and Windows + // ReadDirectoryChangesW are already watching recursively + // from the base path (see `create_debouncer`), and + // registering a second overlapping stream there produces + // duplicate/out-of-order events. + #[cfg(target_os = "linux")] + { + // Register the new directory with the debouncer, then + // drop the mutex BEFORE doing picker-side work — see + // the comment on `BackgroundWatcher::stop` for the + // lock-ordering rationale. + let mut guard = owner_debouncer.lock(); + let Some(debouncer) = guard.as_mut() else { + break; + }; + + if let Err(e) = debouncer.watch(&dir, RecursiveMode::NonRecursive) { + warn!( + ?e, + dir = %dir.display(), + "Failed to init watcher for new directory" + ); + } + } + + track_files_from_new_directories( + &dir, + &strong_picker, + &owner_git_workdir, + &owner_git_worker, + ); + + // Transient strong ref drops here, back + // to weak-only before the next `recv()`. + } + + tracing::info!("Background watcher is stopped"); + }) + .expect("failed to spawn fff-watcher-owner thread"); + + Ok(Self { + debouncer, + watch_tx: Some(watch_tx), + owner_thread: Some(owner_thread), + }) + } + + #[allow(clippy::too_many_arguments)] + fn create_debouncer( + base_path: PathBuf, + git_workdir: Option, + shared_picker: SharedFilePicker, + shared_frecency: SharedFrecency, + mode: FFFMode, + use_recursive: bool, + watch_tx: mpsc::Sender, + git_status_worker: Arc, + ) -> Result { + let config = Config::default() + // do not follow symlinks as then notifiers spawns a bunch of events for symlinked + // files that could be git ignored, we have to property differentiate those and if + // the file was edited through a + .with_follow_symlinks(false) + // only the actual modification events, ignore the open syscals that we can generate by + // our own grep calls and preview window rendering + .with_event_kinds(EventKindMask::CORE); + + let git_workdir_for_handler = git_workdir.clone(); + let base_path_for_handler = base_path.clone(); + let shared_picker_for_watching = shared_picker.clone(); + let file_picker = shared_picker.weaken(); + let mut debouncer = new_debouncer_opt( + DEBOUNCE_TIMEOUT, + Some(DEBOUNCE_TIMEOUT / 2), // tick rate for the event span + { + move |result: DebounceEventResult| match result { + Ok(events) => { + let Some(file_picker) = file_picker.upgrade() else { + return; + }; + + let new_dirs = handle_debounced_events( + mode, + events, + &base_path_for_handler, + &git_workdir_for_handler, + &file_picker, + &shared_frecency, + &git_status_worker, + ); + + // every new directory created has to be reflected in the picker state + for dir in new_dirs { + if let Err(e) = watch_tx.send(dir) { + error!(?e, "Failed to send directory update error"); + } + } + } + Err(errors) => { + error!("File watcher errors: {:?}", errors); + } + } + }, + // There is an issue with recommended cache implementation on macos + // it keeps track of all the files added to the watcher which is not a problem + // for us because any rename to the file will anyway require the removing from the + // ordedred index and adding it back with the new name + NoCache::new(), + config, + )?; + + if use_recursive { + // if the platform supports native watcher recursion + debouncer.watch(base_path.as_path(), RecursiveMode::Recursive)?; + info!( + "File watcher initialized with single recursive watch on {} \ + (exceeded threshold of {})", + base_path.display(), + MAX_MACOS_NONRECURSIVE_WATCHES, + ); + } else { + debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?; + + const MAX_CONSECUTIVE_WATCH_FAILURES: usize = 16; + + let mut watched = 0usize; + let mut consecutive_failures = 0usize; + + // `inotify` is fast-fail: on ENOSPC it returns + // immediately, no kernel retry loop, so holding this lock is free + if let Some(guard) = shared_picker_for_watching.read().ok() + && let Some(picker) = guard.as_ref() + { + use std::ops::ControlFlow; + picker.for_each_dir(|dir| { + match debouncer.watch(dir, RecursiveMode::NonRecursive) { + Ok(()) => { + watched += 1; + consecutive_failures = 0; + ControlFlow::Continue(()) + } + Err(e) => { + consecutive_failures += 1; + if consecutive_failures <= 4 { + warn!("Failed to watch directory {}: {}", dir.display(), e); + } + + if consecutive_failures >= MAX_CONSECUTIVE_WATCH_FAILURES { + warn!( + consecutive_failures, + watched, + "Giving up setting file watcher for all the directories. Check if your system has enough fs watchers limit." + ); + + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + } + }); + } + + tracing::info!( + ?watched, + path = ?base_path.display(), + "File watcher initialized" + ); + } + + // The .git directory is excluded from the file list but we still need + // to observe changes that affect git status (staging, unstaging, + // committing, branch switches, merges, etc) + watch_git_status_paths(&mut debouncer, git_workdir.as_ref()); + + Ok(debouncer) + } + + /// Signals the background watcher threads to shut down, doesn't guarantee to deallocate immediately + pub fn stop(&mut self) { + self.watch_tx.take(); + if let Some(debouncer) = self.debouncer.lock().take() { + debouncer.stop_nonblocking(); + } + + self.owner_thread.take(); + + info!("Background file watcher stop signaled"); + } + + pub(crate) fn request_watch_dir(&self, dir: PathBuf) -> bool { + match self.watch_tx.as_ref() { + Some(tx) => tx.send(dir).is_ok(), + None => false, + } + } +} + +impl Drop for BackgroundWatcher { + fn drop(&mut self) { + self.stop(); + } +} + +#[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency, git_status_worker), level = Level::DEBUG)] +fn handle_debounced_events( + mode: FFFMode, + events: Vec, + base_path: &Path, + git_workdir: &Option, + shared_picker: &SharedFilePicker, + shared_frecency: &SharedFrecency, + git_status_worker: &Arc, +) -> Vec { + // this will be called very often, we have to minimiy the lock time for file picker + let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok()); + // Prefer the walker's own ignore rules (zlob); grab a cheap Arc clone once + // per batch so we don't hold the picker lock during filtering. + let walker_rules = shared_picker + .read() + .ok() + .and_then(|g| g.as_ref().and_then(|p| p.ignore_rules())); + let filter = IgnoreFilter::new(base_path, walker_rules, repo.as_ref()); + let mut need_full_rescan = false; + let mut need_full_git_rescan = false; + let mut paths_to_remove = Vec::new(); + let mut dirs_to_remove: Vec = Vec::new(); + let mut paths_to_add_or_modify = Vec::new(); + let mut new_dirs_to_watch = Vec::new(); + let mut affected_paths_count = 0usize; + + for debounced_event in &events { + // It is very important to not react to the access errors because we inevitably + // gonna trigger the sync by our own preview or other unnecessary noise + if matches!( + debounced_event.event.kind, + EventKind::Access( + AccessKind::Read + | AccessKind::Open(_) + | AccessKind::Close(AccessMode::Read | AccessMode::Execute) + ) + ) { + continue; + } + + // When macOS FSEvents (or other backends) overflow their event buffer, the kernel + // drops individual events and emits a rescan flag telling us to re-scan the subtree + if debounced_event.event.need_rescan() { + if debounced_event.event.paths.len() < 16 // this should be usually one event + && debounced_event + .paths + .iter() + // but we are smart enough and not falling into the paths + .all(|p| should_include_file(p, &filter)) + { + break; + } + + warn!( + "Received rescan event for paths {:?}, triggering full rescan", + debounced_event.event.paths + ); + need_full_rescan = true; + break; + } + + tracing::debug!(event = ?debounced_event.event, "Processing FS event"); + for path in &debounced_event.event.paths { + if matches!( + path.file_name().and_then(|f| f.to_str()), + Some(".ignore") | Some(".gitignore") + ) { + info!( + "Detected change in ignore definition file: {}", + path.display() + ); + + need_full_rescan = true; + break; + } + + if is_dotgit_change_affecting_status(path, &repo) { + need_full_git_rescan = true; + } + + if is_git_file(path) { + continue; + } + + // Use a combination of event kind and filesystem state to decide + // whether a path is an addition/modification or a removal. + // + // We cannot rely on `path.exists()` alone because: + // - A freshly created file might not be visible yet (race). + // - macOS FSEvents uses Modify(Name(Any)) for both rename-in + // and rename-out, so we must stat the path to disambiguate. + // + // We cannot rely on event kind alone because: + // - Remove events are not always emitted (macOS often sends + // Modify(Name(Any)) instead of Remove). + let is_removal = matches!(debounced_event.event.kind, EventKind::Remove(_)); + + // Directory-level remove: both fsevents and inotify delivers a single + // `Remove(Folder)` event for a whole directory tree (e.g. + // after `git reset --hard` wipes a dir full of staged-but- + // uncommitted files). + let is_folder_removal = matches!( + debounced_event.event.kind, + EventKind::Remove(notify::event::RemoveKind::Folder) + ); + + if is_folder_removal { + dirs_to_remove.push(path.to_path_buf()); + } else if is_removal || !path.exists() { + paths_to_remove.push(path.as_path()); + } else if path.is_dir() { + if !is_path_ignored(path, &filter) { + new_dirs_to_watch.push(path.to_path_buf()); + } + } else { + // For additions/modifications, still filter gitignored files. + if should_include_file(path, &filter) { + paths_to_add_or_modify.push(path.as_path()); + } + } + } + + affected_paths_count += debounced_event.event.paths.len(); + if affected_paths_count > MAX_OVERFLOW_FILES { + warn!( + ?affected_paths_count, + max = MAX_OVERFLOW_FILES, + "Too many affected paths in a single batch, triggering full rescan", + ); + + need_full_rescan = true; + break; + } + + if need_full_rescan { + break; + } + } + + if need_full_rescan { + info!(?affected_paths_count, "Triggering full rescan"); + if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) { + error!("Failed to trigger full rescan: {:?}", e); + } + return Vec::new(); + } + + // It's important to get the allocated sort + sort_with_buffer(paths_to_add_or_modify.as_mut_slice(), |a, b| { + a.as_os_str().cmp(b.as_os_str()) + }); + paths_to_add_or_modify.dedup_by(|a, b| a.as_os_str().eq(b.as_os_str())); + + info!( + "Event processing summary: {} to remove, {} dirs to remove, {} to add/modify, {} new dirs", + paths_to_remove.len(), + dirs_to_remove.len(), + paths_to_add_or_modify.len(), + new_dirs_to_watch.len() + ); + + if paths_to_remove.is_empty() + && dirs_to_remove.is_empty() + && paths_to_add_or_modify.is_empty() + && !need_full_git_rescan + { + debug!("No file index changes to apply"); + return new_dirs_to_watch; + } + + let mut files_to_update_git_status = Vec::new(); + let mut need_full_rescan = false; + let mut overflow_count = 0; + + if !paths_to_remove.is_empty() + || !dirs_to_remove.is_empty() + || !paths_to_add_or_modify.is_empty() + { + debug!( + "Applying file index changes: {} to remove, {} dirs to remove, {} to add/modify", + paths_to_remove.len(), + dirs_to_remove.len(), + paths_to_add_or_modify.len(), + ); + + let Ok(mut guard) = shared_picker.write() else { + error!("Failed to acquire file picker write lock"); + return new_dirs_to_watch; + }; + let Some(ref mut picker) = *guard else { + error!("File picker not initialized"); + return new_dirs_to_watch; + }; + + for dir in &dirs_to_remove { + let count = picker.remove_all_files_in_dir(dir); + debug!("remove_all_files_in_dir({:?}) -> {} files", dir, count); + } + + for path in &paths_to_remove { + let removed = picker.remove_file_by_path(path); + debug!("remove_file_by_path({:?}) -> {}", path, removed); + } + + files_to_update_git_status.reserve(paths_to_add_or_modify.len()); + for path in &paths_to_add_or_modify { + if picker.handle_create_or_modify(path).is_some() { + files_to_update_git_status.push(path.to_path_buf()); + } else { + need_full_rescan = true; + } + } + + overflow_count = picker.get_overflow_files().len(); + } + + info!( + files_updated = files_to_update_git_status.len(), + overflow_count, "File index changes applied", + ); + if need_full_rescan || overflow_count > MAX_OVERFLOW_FILES { + info!("Watcher faced limit of index overflow. Triggering rescan"); + if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) { + error!("Failed to trigger full rescan: {:?}", e); + } + } + + // AI mode: auto-track frecency for all modified/created files. + // Uses a 5-minute cooldown per file to prevent score inflation from rapid + // burst edits (AI agents often edit the same file many times in minutes). + // This runs after apply_changes so the picker write lock is released. + if mode.is_ai() && !paths_to_add_or_modify.is_empty() { + let mut tracked_count = 0usize; + if let Ok(frecency_guard) = shared_frecency.read() + && let Some(ref frecency) = *frecency_guard + { + for path in &paths_to_add_or_modify { + // Skip if this file was tracked less than 5 minutes ago + let should_track = match frecency.seconds_since_last_access(path) { + Ok(Some(secs)) => secs >= AI_MODE_COOLDOWN_SECS, + Ok(None) => true, // Never tracked before + Err(_) => true, // DB error, track anyway + }; + if !should_track { + continue; + } + + if let Err(e) = frecency.track_access(path) { + error!("Failed to track frecency for {:?}: {:?}", path, e); + } else { + tracked_count += 1; + } + } + if tracked_count > 0 { + info!("AI mode: tracked frecency for {} files", tracked_count); + } + } + + // Update in-memory frecency scores for tracked files + if tracked_count > 0 + && let Ok(mut picker_guard) = shared_picker.write() + && let Some(ref mut picker) = *picker_guard + && let Ok(frecency_guard) = shared_frecency.read() + && let Some(ref frecency) = *frecency_guard + { + for path in &paths_to_add_or_modify { + let _ = picker.update_single_file_frecency(path, frecency); + } + } + } + + // do not try to update the paths if we anyway going to rescan everything from scratch + // no repo => no consumer thread, so don't accumulate paths nobody will drain + if !need_full_rescan && repo.is_some() { + if need_full_git_rescan { + // A full git rescan re-reads every tracked path (including ones that just + // went clean after a commit), so it already subsumes the per-path update. + git_status_worker.request_full_rescan(); + } else if !files_to_update_git_status.is_empty() { + git_status_worker.enqueue_paths(files_to_update_git_status); + } + } + + new_dirs_to_watch +} + +/// After registering a watch on a newly created directory, list its +/// immediate children and add any files to the picker. +fn track_files_from_new_directories( + dir: &Path, + shared_picker: &SharedFilePicker, + git_workdir: &Option, + git_status_worker: &Arc, +) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + + let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok()); + // Prefer the walker's ignore rules; read base_path + rules from the picker. + let (base_path, walker_rules) = match shared_picker.read().ok().and_then(|g| { + g.as_ref() + .map(|p| (p.base_path().to_path_buf(), p.ignore_rules())) + }) { + Some(pair) => pair, + None => return, + }; + + let filter = IgnoreFilter::new(&base_path, walker_rules, repo.as_ref()); + let mut files_to_add = Vec::new(); + + for entry in entries.flatten() { + if entry.file_type().is_ok_and(|ft| ft.is_file()) { + let path = entry.path(); + if should_include_file(&path, &filter) { + files_to_add.push(path); + } + } + } + + if files_to_add.is_empty() { + return; + } + + let added = files_to_add.len(); + + { + let Ok(mut guard) = shared_picker.write() else { + return; + }; + + let Some(ref mut picker) = *guard else { + return; + }; + + for path in &files_to_add { + picker.handle_create_or_modify(path); + } + } + + if repo.is_some() { + git_status_worker.enqueue_paths(files_to_add); + } + + debug!( + "Injected {} existing files from new directory {}", + added, + dir.display(), + ); +} + +fn should_include_file(path: &Path, filter: &IgnoreFilter) -> bool { + // Directories are not indexed — only regular files (and symlinks to files). + if path.is_dir() { + return false; + } + !filter.is_ignored(path) +} + +#[inline] +fn is_path_ignored(path: &Path, filter: &IgnoreFilter) -> bool { + filter.is_ignored(path) +} + +struct IgnoreFilter<'a> { + base_path: &'a Path, + /// Reusable ignore rules from the last walk (zlob backend only). + rules: Option>, + /// libgit2 repo, consulted only when `rules` is `None`. Borrowed from the + /// caller's repo (also used for git-status queries) to avoid re-opening. + repo: Option<&'a Repository>, +} + +impl<'a> IgnoreFilter<'a> { + fn new( + base_path: &'a Path, + rules: Option>, + repo: Option<&'a Repository>, + ) -> Self { + Self { + base_path, + rules, + repo, + } + } + + /// Whether `path` (absolute) is ignored. + fn is_ignored(&self, path: &Path) -> bool { + if let Some(rules) = self.rules.as_ref() { + let Ok(rel) = path.strip_prefix(self.base_path) else { + return false; + }; + // `IgnoreRules::is_ignored` enumerates every ancestor .gitignore + // layer internally, so a leaf under an ignored directory (rule + // `build/`, path `build/out.rs`) is caught in one call. + return rules.is_ignored(rel); + } + match self.repo { + Some(repo) => repo.is_path_ignored(path) == Ok(true), + // No repo and no rules: fall back to the non-code-dir heuristic. + None => crate::ignore::is_non_code_directory(path), + } + } +} + +#[inline] +pub(crate) fn is_git_file(path: &Path) -> bool { + // it could be in submodule + path.components() + .any(|component| component.as_os_str() == ".git") +} + +fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option) -> bool { + let Some(repo) = repo.as_ref() else { + return false; + }; + + let git_dir = repo.path(); + + if let Ok(path_in_git_dir) = changed.strip_prefix(git_dir) { + // Only react to changes that rewrite the worktree state: commits, + // staging, checkouts, merges, conflict resolution. Ref-only updates + // under refs/ (fetch, push, tag writes, pack-refs) do not change + // which files are modified/untracked, so we deliberately skip them + // watching refs/ recursively would cost one inotify watch per ref + // namespace on repos with many branches/remotes. + if path_in_git_dir == Path::new("index") || path_in_git_dir == Path::new("index.lock") { + return true; + } + + if path_in_git_dir == Path::new("HEAD") { + return true; + } + + // some of the git ops are not involving nethier index nor HEAD change, or sometimes + // index updates can arrive too late after the change - that's why we track the log + // the actual user action, once user + if path_in_git_dir == Path::new("logs/HEAD") { + return true; + } + + if let Some(fname) = path_in_git_dir.file_name().and_then(|f| f.to_str()) + && matches!(fname, "MERGE_HEAD" | "CHERRY_PICK_HEAD" | "REVERT_HEAD") + { + return true; + } + } + + false +} + +fn watch_git_status_paths(debouncer: &mut Debouncer, git_workdir: Option<&PathBuf>) { + let Some(workdir) = git_workdir else { + return; + }; + + let git_dir = workdir.join(".git"); + if !git_dir.is_dir() { + return; + } + + // We have tried to be smart about the internal git state but + // it appeared more harmful that it's worth it, so we just watch + // for the most obvious paths like HEAD, MERGE_HEAD, index.lock + if let Err(e) = debouncer.watch(&git_dir, RecursiveMode::NonRecursive) { + warn!("Failed to watch .git directory: {}", e); + } + + // `.git` above is non-recursive, so on Linux (per-dir inotify watches) + // events for `logs/HEAD` — the commit-finished signal used by + // `is_dotgit_change_affecting_status` — would never be delivered without + // watching `.git/logs` itself. On macOS/Windows the recursive base watch + // already covers it; an extra watch is harmless there. + let logs_dir = git_dir.join("logs"); + if logs_dir.is_dir() + && let Err(e) = debouncer.watch(&logs_dir, RecursiveMode::NonRecursive) + { + warn!("Failed to watch .git/logs directory: {}", e); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dotgit_status_filter_matches_worktree_state_changes() { + let tmp = tempfile::tempdir().unwrap(); + let repo = git2::Repository::init(tmp.path()).unwrap(); + let git_dir = repo.path().to_path_buf(); + let repo = Some(repo); + + let affecting = ["index", "index.lock", "HEAD", "logs/HEAD", "MERGE_HEAD"]; + for p in affecting { + assert!( + is_dotgit_change_affecting_status(&git_dir.join(p), &repo), + "{p} must trigger a git status rescan" + ); + } + + // Ref-only updates (fetch/push/tags) and commit scratch files must not. + let non_affecting = [ + "refs/heads/main", + "refs/heads/main.lock", + "logs/refs/remotes/origin/main", + "COMMIT_EDITMSG", + "packed-refs", + ]; + for p in non_affecting { + assert!( + !is_dotgit_change_affecting_status(&git_dir.join(p), &repo), + "{p} must NOT trigger a git status rescan" + ); + } + + // Worktree paths outside .git never match. + assert!(!is_dotgit_change_affecting_status( + &tmp.path().join("src/main.rs"), + &repo + )); + assert!(!is_dotgit_change_affecting_status( + &git_dir.join("index"), + &None + )); + } +} diff --git a/crates/fff-core/src/bigram_filter.rs b/crates/fff-core/src/bigram_filter.rs new file mode 100644 index 0000000..6ce2a31 --- /dev/null +++ b/crates/fff-core/src/bigram_filter.rs @@ -0,0 +1,1227 @@ +use crate::constants::MAX_INDEXABLE_FILE_SIZE; +use ahash::AHashMap; +use rayon::iter::{IndexedParallelIterator, ParallelIterator}; +use rayon::slice::ParallelSlice; +use std::cell::UnsafeCell; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering}; + +use crate::{FileItem, constants}; + +/// Maximum number of distinct bigrams tracked in the inverted index. +/// 95 printable ASCII chars (32..=126) after lowercasing → ~70 distinct → 4900 possible. +/// We cap at 5000 to cover all printable bigrams with margin. +/// 5000 columns × 62.5KB (500k files) = 305MB. For 50k files: 30MB. +const MAX_BIGRAM_COLUMNS: usize = 5000; + +/// Sentinel value: bigram has no allocated column. +const NO_COLUMN: u16 = u16::MAX; + +/// 1024 × u64 = 8 KB covers all 65536 possible bigram keys. +const SEEN_WORDS: usize = 1024; + +/// Content size where the branchless two-pass `add_long_content` overtakes +/// the single-pass `add_short_content`: ~-35% on 4 KB files, but its fixed +/// flush scan dominates files under ~1 KB. See bigram_bench `bigram_build`. +const LONG_CONTENT_MIN_LEN: usize = 1024; + +thread_local! { + static NORM_BUF: std::cell::RefCell> = + std::cell::RefCell::new(Vec::with_capacity(4096)); +} + +/// Temporary sync dense builder for the bigram index. +/// Builds from the many threads reading file contents in parallel +pub struct BigramIndexBuilder { + // we use lookup as atomics only in the builder because it is filled by the rayon threads + // the actual index uses pure u16 for the allocations + lookup: Vec, + /// Flat bitset data, materialised on first use. + col_data: OnceLock>>, + next_column: AtomicU16, + words: usize, + file_count: usize, + populated: AtomicUsize, +} + +// SAFETY: `col_data`'s interior mutability is coordinated via disjoint +// `word_idx` ranges (word-aligned file partitioning in the driver), so +// concurrent access is safe despite the `UnsafeCell`. See builder doc. +unsafe impl Sync for BigramIndexBuilder {} + +impl BigramIndexBuilder { + pub fn new(file_count: usize) -> Self { + let words = file_count.div_ceil(64); + let mut lookup = Vec::with_capacity(65536); + lookup.resize_with(65536, || AtomicU16::new(NO_COLUMN)); + Self { + lookup, + col_data: OnceLock::new(), + next_column: AtomicU16::new(0), + words, + file_count, + populated: AtomicUsize::new(0), + } + } + + /// Lazily materialise the full `MAX_BIGRAM_COLUMNS * words` bitset + /// on first access. + #[inline(always)] + fn col_data_cell(&self) -> &UnsafeCell> { + self.col_data.get_or_init(|| { + let total = MAX_BIGRAM_COLUMNS * self.words; + UnsafeCell::new(vec![0u64; total].into_boxed_slice()) + }) + } + + /// Raw pointer to the start of the bitset slab. Used for in-place + /// `|=` writes under the partitioning invariant. + #[inline(always)] + fn col_data_ptr(&self) -> *mut u64 { + unsafe { (*self.col_data_cell().get()).as_mut_ptr() } + } + + #[inline] + fn get_or_alloc_column(&self, key: u16) -> u16 { + let current = self.lookup[key as usize].load(Ordering::Relaxed); + if current != NO_COLUMN { + return current; + } + let new_col = self.next_column.fetch_add(1, Ordering::Relaxed); + if new_col >= MAX_BIGRAM_COLUMNS as u16 { + return NO_COLUMN; + } + + match self.lookup[key as usize].compare_exchange( + NO_COLUMN, + new_col, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => new_col, + Err(existing) => existing, + } + } + + /// Test/bench accessor for a column's raw bitset words. Assumes the + /// caller has joined all writers (no concurrent mutation). + #[cfg(test)] + fn column_bitset(&self, col: u16) -> &[u64] { + let start = col as usize * self.words; + let slab = unsafe { &*self.col_data_cell().get() }; + &slab[start..start + self.words] + } + + #[doc(hidden)] // `pub` (via `#[doc(hidden)]`) only for benchmarking + pub fn add_file_content(&self, skip_builder: &Self, file_idx: usize, content: &[u8]) { + if content.len() < 2 { + return; + } + + debug_assert!(file_idx < self.file_count); + let word_idx = file_idx / 64; + let bit_mask = 1u64 << (file_idx % 64); + + NORM_BUF.with_borrow_mut(|buf| { + let len = content.len(); + if buf.len() < len { + buf.resize(len.next_power_of_two().max(4096), 0); + } + + normalize_bytes(content, &mut buf[..len]); + let n = &buf[..len]; + + // Both paths record the identical bigram set; the split exists + // purely for speed (see LONG_CONTENT_MIN_LEN). + if len >= LONG_CONTENT_MIN_LEN { + self.add_long_content(skip_builder, n, word_idx, bit_mask); + } else { + self.add_short_content(skip_builder, n, word_idx, bit_mask); + } + }); + + self.populated.fetch_add(1, Ordering::Relaxed); + skip_builder.populated.fetch_add(1, Ordering::Relaxed); + } + + // Branchless two-pass: set every pair in stack-local bitmaps, including + // pairs touching the 0 sentinel — flush_seen masks those out. ~-35% vs + // the single pass on 4 KB files. + #[inline(always)] + fn add_long_content(&self, skip_builder: &Self, n: &[u8], word_idx: usize, bit_mask: u64) { + // Stack-local dedup bitsets: 1024 × u64 = 8 KB each, covers all 65536 + // bigram keys. Has to fit in L1 cache. + let mut seen_consec = [0u64; SEEN_WORDS]; + let mut seen_skip = [0u64; SEEN_WORDS]; + + let mut n0 = n[0]; + let mut n1 = n[1]; + + let key = (n0 as usize) << 8 | n1 as usize; + // SAFETY: key < 65536, so key >> 6 < 1024 = SEEN_WORDS. + unsafe { *seen_consec.get_unchecked_mut(key >> 6) |= 1u64 << (key & 63) }; + + for &cur in &n[2..] { + let ck = (n1 as usize) << 8 | cur as usize; + let sk = (n0 as usize) << 8 | cur as usize; + unsafe { + *seen_consec.get_unchecked_mut(ck >> 6) |= 1u64 << (ck & 63); + *seen_skip.get_unchecked_mut(sk >> 6) |= 1u64 << (sk & 63); + } + + n0 = n1; + n1 = cur; + } + + self.flush_seen(&seen_consec, word_idx, bit_mask); + skip_builder.flush_seen(&seen_skip, word_idx, bit_mask); + } + + #[inline(always)] + fn add_short_content(&self, skip_builder: &Self, n: &[u8], word_idx: usize, bit_mask: u64) { + let mut seen_consec = [0u64; SEEN_WORDS]; + let mut seen_skip = [0u64; SEEN_WORDS]; + + let consec_base = self.col_data_ptr(); + let consec_words = self.words; + let skip_base = skip_builder.col_data_ptr(); + let skip_words = skip_builder.words; + + let mut n0 = n[0]; + let mut n1 = n[1]; + + if n0 != 0 && n1 != 0 { + let key = (n0 as u16) << 8 | n1 as u16; + self.record_bigram( + &mut seen_consec, + key, + word_idx, + bit_mask, + consec_base, + consec_words, + ); + } + + for &cur in &n[2..] { + if cur != 0 { + if n1 != 0 { + let key = (n1 as u16) << 8 | cur as u16; + self.record_bigram( + &mut seen_consec, + key, + word_idx, + bit_mask, + consec_base, + consec_words, + ); + } + if n0 != 0 { + let key = (n0 as u16) << 8 | cur as u16; + skip_builder.record_bigram( + &mut seen_skip, + key, + word_idx, + bit_mask, + skip_base, + skip_words, + ); + } + } + n0 = n1; + n1 = cur; + } + } + + #[inline(always)] + fn record_bigram( + &self, + seen: &mut [u64; SEEN_WORDS], + key: u16, + word_idx: usize, + bit_mask: u64, + col_base: *mut u64, + words: usize, + ) { + let k = key as usize; + let w = k >> 6; + let bit = 1u64 << (k & 63); + // SAFETY: w = key/64 with key: u16, so w < 1024 = SEEN_WORDS. + let prev = unsafe { *seen.get_unchecked(w) }; + if prev & bit == 0 { + unsafe { + *seen.get_unchecked_mut(w) = prev | bit; + } + let col = self.get_or_alloc_column(key); + if col != NO_COLUMN { + unsafe { + let p = col_base.add(col as usize * words + word_idx); + *p |= bit_mask; + } + } + } + } + + fn flush_seen(&self, seen: &[u64; SEEN_WORDS], word_idx: usize, bit_mask: u64) { + let col_base = self.col_data_ptr(); + let words = self.words; + for (blk, block) in seen.chunks_exact(8).enumerate() { + // OR-test whole blocks so the mostly-empty bitmap scans fast. + if block.iter().fold(0u64, |a, &w| a | w) == 0 { + continue; + } + for (j, &word_bits) in block.iter().enumerate() { + let w = blk * 8 + j; + let mut bits = match w & 3 { + _ if w < 4 => 0, + 0 => word_bits & !1, + _ => word_bits, + }; + while bits != 0 { + let key = (w << 6 | bits.trailing_zeros() as usize) as u16; + bits &= bits - 1; + let col = self.get_or_alloc_column(key); + if col != NO_COLUMN { + unsafe { + let p = col_base.add(col as usize * words + word_idx); + *p |= bit_mask; + } + } + } + } + } + } + + pub fn is_ready(&self) -> bool { + self.populated.load(Ordering::Relaxed) > 0 + } + + pub fn columns_used(&self) -> u16 { + self.next_column + .load(Ordering::Relaxed) + .min(MAX_BIGRAM_COLUMNS as u16) + } + + /// Compress the dense builder into a compact `BigramFilter`. + /// + /// Retains columns where the bigram appears in ≥`min_density_pct`% (or + /// the default ~3.1% heuristic when `None`) and <90% of indexed files. + /// Sparse columns carry too little data to justify their memory; + /// ubiquitous columns (≥90%) are nearly all-ones and barely filter. + #[inline(always)] + pub fn compress(self, min_density_pct: Option) -> BigramFilter { + let cols = self.columns_used() as usize; + let words = self.words; + let file_count = self.file_count; + let populated = self.populated.load(Ordering::Relaxed); + let dense_bytes = words * 8; // cost of one dense column + + let old_lookup = self.lookup; + // If no file ever populated content, col_data was never + // materialised. Treat as empty — every column falls through. + let col_data: Option> = self.col_data.into_inner().map(UnsafeCell::into_inner); + + let mut lookup: Vec = vec![NO_COLUMN; 65536]; + let mut dense_data: Vec = Vec::with_capacity(cols * words); + let mut dense_count: usize = 0; + + if let Some(col_data) = col_data.as_deref() { + for key in 0..65536usize { + let old_col = old_lookup[key].load(Ordering::Relaxed); + if old_col == NO_COLUMN || old_col as usize >= cols { + continue; + } + + let col_start = old_col as usize * words; + let bitset = &col_data[col_start..col_start + words]; + + // count set bits to decide if this column is worth keeping. + let mut popcount = 0u32; + for &word in bitset.iter().take(words) { + popcount += word.count_ones(); + } + + // drop bigrams appearing in too few files + let not_to_rare = if let Some(min_pct) = min_density_pct { + // Percentage-based: require ≥ min_pct% of populated files. + populated > 0 && (popcount as usize) * 100 >= populated * min_pct as usize + } else { + // Default: popcount ≥ words × 2 (~3.1% of files). + (popcount as usize * 4) >= dense_bytes + }; + + if !not_to_rare { + continue; + } + + // Drop ubiquitous bigrams — columns ≥90% ones carry almost no + // filtering power and just waste memory + AND cycles. + if populated > 0 && (popcount as usize) * 10 >= populated * 9 { + continue; + } + + let dense_idx = dense_count as u16; + lookup[key] = dense_idx; + dense_count += 1; + + dense_data.extend_from_slice(bitset); + } + } + + BigramFilter { + lookup, + dense_data, + dense_count, + words, + file_count, + populated, + skip_index: None, + } + } +} + +unsafe impl Send for BigramIndexBuilder {} + +/// Inverted bigram index with optional "skip-1" extension +/// Copmressed into bitset for minimal usage, the layout of this struct actually matters +#[derive(Debug)] +pub struct BigramFilter { + lookup: Vec, + /// Flat buffer of all dense column data laid out at fixed stride `words`. + /// Column `i` starts at `i * words`. + dense_data: Vec, // do not try to change this to u8 it has to be wordsize + dense_count: usize, + words: usize, + file_count: usize, + populated: usize, + /// Optional skip-1 bigram index (stride 2). Built from character pairs + /// at distance 2, e.g. "ABCDE" → (A,C),(B,D),(C,E). ANDead with the + /// consecutive bigram candidates during query to dramatically reduce + /// false positives. + skip_index: Option>, +} + +/// SIMD-friendly bitwise AND of two equal-length bitsets. +// Auto vectorized (don't touch) +#[inline] +fn bitset_and(result: &mut [u64], bitset: &[u64]) { + result + .iter_mut() + .zip(bitset.iter()) + .for_each(|(r, b)| *r &= *b); +} + +impl BigramFilter { + /// AND the posting lists for all query bigrams (consecutive + skip). + /// Returns None if no query bigrams are tracked. + pub fn query(&self, pattern: &[u8]) -> Option> { + if pattern.len() < 2 { + return None; + } + + let mut result = vec![u64::MAX; self.words]; + if !self.file_count.is_multiple_of(64) { + let last = self.words - 1; + result[last] = (1u64 << (self.file_count % 64)) - 1; + } + + let words = self.words; + let mut has_filter = false; + + let mut prev = pattern[0]; + for &b in &pattern[1..] { + if (32..=126).contains(&prev) && (32..=126).contains(&b) { + let key = (prev.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16; + let col = self.lookup[key as usize]; + if col != NO_COLUMN { + let offset = col as usize * words; + // SAFETY: compress() guarantees offset + words <= dense_data.len() + let slice = unsafe { self.dense_data.get_unchecked(offset..offset + words) }; + bitset_and(&mut result, slice); + has_filter = true; + } + } + prev = b; + } + + // strid-1 bigrams + if let Some(skip) = &self.skip_index + && pattern.len() >= 3 + && let Some(skip_candidates) = skip.query_skip(pattern) + { + bitset_and(&mut result, &skip_candidates); + has_filter = true; + } + + has_filter.then_some(result) + } + + /// Query using stride-2 bigrams from the pattern. + /// For "ABCDE" queries with keys (A,C), (B,D), (C,E). + fn query_skip(&self, pattern: &[u8]) -> Option> { + let mut result = vec![u64::MAX; self.words]; + if !self.file_count.is_multiple_of(64) { + let last = self.words - 1; + result[last] = (1u64 << (self.file_count % 64)) - 1; + } + + let words = self.words; + let mut has_filter = false; + + for i in 0..pattern.len().saturating_sub(2) { + let a = pattern[i]; + let b = pattern[i + 2]; + if (32..=126).contains(&a) && (32..=126).contains(&b) { + let key = (a.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16; + let col = self.lookup[key as usize]; + if col != NO_COLUMN { + let offset = col as usize * words; + let slice = unsafe { self.dense_data.get_unchecked(offset..offset + words) }; + bitset_and(&mut result, slice); + has_filter = true; + } + } + } + + has_filter.then_some(result) + } + + /// Attach a skip-1 bigram index for tighter candidate filtering. + pub fn set_skip_index(&mut self, skip: BigramFilter) { + self.skip_index = Some(Box::new(skip)); + } + + #[inline] + pub fn is_candidate(candidates: &[u64], file_idx: usize) -> bool { + let word = file_idx / 64; + let bit = file_idx % 64; + word < candidates.len() && candidates[word] & (1u64 << bit) != 0 + } + + pub fn count_candidates(candidates: &[u64]) -> usize { + candidates.iter().map(|w| w.count_ones() as usize).sum() + } + + pub fn is_ready(&self) -> bool { + self.populated > 0 + } + + pub fn file_count(&self) -> usize { + self.file_count + } + + pub fn columns_used(&self) -> usize { + self.dense_count + } + + /// Total heap bytes used by this index (lookup + dense data + skip). + pub fn heap_bytes(&self) -> usize { + let lookup_bytes = self.lookup.len() * std::mem::size_of::(); + let dense_bytes = self.dense_data.len() * std::mem::size_of::(); + let skip_bytes = self.skip_index.as_ref().map_or(0, |s| s.heap_bytes()); + lookup_bytes + dense_bytes + skip_bytes + } + + /// Check whether a bigram key is present in this index. + pub fn has_key(&self, key: u16) -> bool { + self.lookup[key as usize] != NO_COLUMN + } + + /// Raw lookup table (65536 entries mapping bigram key → column index). + pub fn lookup(&self) -> &[u16] { + &self.lookup + } + + /// Flat dense bitset data at fixed stride `words`. + pub fn dense_data(&self) -> &[u64] { + &self.dense_data + } + + /// Number of u64 words per column (= ceil(file_count / 64)). + pub fn words(&self) -> usize { + self.words + } + + /// Number of dense columns retained after compression. + pub fn dense_count(&self) -> usize { + self.dense_count + } + + /// Number of files that contributed content to the index. + pub fn populated(&self) -> usize { + self.populated + } + + /// Reference to the optional skip-1 bigram sub-index. + pub fn skip_index(&self) -> Option<&BigramFilter> { + self.skip_index.as_deref() + } + + /// Create a new bigram filter from the internal data + pub fn new( + lookup: Vec, + dense_data: Vec, + dense_count: usize, + words: usize, + file_count: usize, + populated: usize, + ) -> Self { + Self { + lookup, + dense_data, + dense_count, + words, + file_count, + populated, + skip_index: None, + } + } +} + +/// Single-byte normalize: 0 for non-printable, lowercased byte otherwise. +/// 0 is a safe sentinel: lowered printable bytes are 32..=126. +#[inline(always)] +fn normalize_byte_scalar(b: u8) -> u8 { + let printable = b.wrapping_sub(32) <= 94; + let lower = b | ((b.wrapping_sub(b'A') < 26) as u8 * 0x20); + if printable { lower } else { 0 } +} + +/// Bulk version: write `dst[i]` = `normalize_byte_scalar(src[i])` for `i` +/// in `0..src.len()`. Inlined-scalar so LLVM auto-vectorises with the +/// build's baseline SIMD; on x86_64 we runtime-dispatch to AVX2. +/// Caller guarantees `dst.len() >= src.len()`. +#[inline(always)] +fn normalize_bytes(src: &[u8], dst: &mut [u8]) { + debug_assert!(dst.len() >= src.len()); + #[cfg(target_arch = "x86_64")] + { + if std::is_x86_feature_detected!("avx2") { + unsafe { normalize_bytes_avx2(src, dst) }; + return; + } + } + + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + unsafe { normalize_bytes_neon(src, dst) }; + return; + } + + #[allow(unused)] + normalize_bytes_scalar(src, dst); +} + +#[inline(always)] +fn normalize_bytes_scalar(src: &[u8], dst: &mut [u8]) { + for (i, &b) in src.iter().enumerate() { + dst[i] = normalize_byte_scalar(b); + } +} + +/// AVX2 normalize: 32 bytes/iter. AVX2 only has signed cmp, so unsigned +/// range checks use `min(max(v, lo), hi) == v`. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn normalize_bytes_avx2(src: &[u8], dst: &mut [u8]) { + use std::arch::x86_64::*; + let len = src.len(); + let mut i = 0; + let p_lo = _mm256_set1_epi8(32); + let p_hi = _mm256_set1_epi8(126u8 as i8); + let u_lo = _mm256_set1_epi8(b'A' as i8); + let u_hi = _mm256_set1_epi8(b'Z' as i8); + let or20 = _mm256_set1_epi8(0x20); + while i + 32 <= len { + unsafe { + let v = _mm256_loadu_si256(src.as_ptr().add(i) as *const __m256i); + // printable_mask: v in [32, 126] + let clamp_p = _mm256_min_epu8(_mm256_max_epu8(v, p_lo), p_hi); + let printable = _mm256_cmpeq_epi8(v, clamp_p); + // is_upper_mask: v in [65, 90] + let clamp_u = _mm256_min_epu8(_mm256_max_epu8(v, u_lo), u_hi); + let is_upper = _mm256_cmpeq_epi8(v, clamp_u); + let or_bits = _mm256_and_si256(is_upper, or20); + let lower = _mm256_or_si256(v, or_bits); + let out = _mm256_and_si256(lower, printable); + _mm256_storeu_si256(dst.as_mut_ptr().add(i) as *mut __m256i, out); + } + i += 32; + } + while i < len { + dst[i] = normalize_byte_scalar(src[i]); + i += 1; + } +} + +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[target_feature(enable = "neon")] +unsafe fn normalize_bytes_neon(src: &[u8], dst: &mut [u8]) { + use std::arch::aarch64::*; + let len = src.len(); + let mut i = 0; + let v32 = vdupq_n_u8(32); + let v127 = vdupq_n_u8(127); + let va = vdupq_n_u8(b'A'); + let vz1 = vdupq_n_u8(b'Z' + 1); + let v20 = vdupq_n_u8(0x20); + + while i + 16 <= len { + unsafe { + let v = vld1q_u8(src.as_ptr().add(i)); + // printable: v >= 32 AND v < 127 + let ge32 = vcgeq_u8(v, v32); + let lt127 = vcltq_u8(v, v127); + let print_mask = vandq_u8(ge32, lt127); + // is_upper: v >= 'A' AND v < 'Z'+1 + let ge_a = vcgeq_u8(v, va); + let lt_z1 = vcltq_u8(v, vz1); + let upper_mask = vandq_u8(ge_a, lt_z1); + let or_bits = vandq_u8(upper_mask, v20); + let lower = vorrq_u8(v, or_bits); + let out = vandq_u8(lower, print_mask); + + vst1q_u8(dst.as_mut_ptr().add(i), out); + } + i += 16; + } + while i < len { + dst[i] = normalize_byte_scalar(src[i]); + i += 1; + } +} + +pub fn extract_bigrams(content: &[u8]) -> Vec { + if content.len() < 2 { + return Vec::new(); + } + // Use a flat bitset (65536 bits = 8 KB) for dedup — faster than HashSet. + let mut seen = vec![0u64; 1024]; // 1024 * 64 = 65536 bits + let mut bigrams = Vec::new(); + + let mut prev = content[0]; + for &b in &content[1..] { + if (32..=126).contains(&prev) && (32..=126).contains(&b) { + let key = (prev.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16; + let word = key as usize / 64; + let bit = 1u64 << (key as usize % 64); + if seen[word] & bit == 0 { + seen[word] |= bit; + bigrams.push(key); + } + } + prev = b; + } + bigrams +} + +/// Modified and added files store their own bigram sets. Deleted files are +/// tombstoned in a bitset so they can be excluded from base query results. +/// This overlay is updated by the background watcher on every file event +/// and cleared when the base index is rebuilt. +#[derive(Debug)] +pub struct BigramOverlay { + /// Per-file bigram sets for files modified since the base was built. + /// Key = file index in the base `Vec`. + modified: AHashMap>, + + /// Tombstone bitset — one bit per base file. Set bits are excluded + /// from base query results. + tombstones: Vec, + + /// Original files count this overlay was created for. + base_file_count: usize, +} + +impl BigramOverlay { + pub(crate) fn new(base_file_count: usize) -> Self { + let words = base_file_count.div_ceil(64); + Self { + modified: AHashMap::new(), + tombstones: vec![0u64; words], + base_file_count, + } + } + + pub(crate) fn modify_file(&mut self, file_idx: usize, content: &[u8]) { + self.modified.insert(file_idx, extract_bigrams(content)); + } + + pub(crate) fn delete_file(&mut self, file_idx: usize) { + if file_idx < self.base_file_count { + let word = file_idx / 64; + self.tombstones[word] |= 1u64 << (file_idx % 64); + } + self.modified.remove(&file_idx); + } + + /// Return base file indices of modified files whose bigrams match ALL + /// of the given `pattern_bigrams`. + pub(crate) fn query_modified(&self, pattern_bigrams: &[u16]) -> Vec { + if pattern_bigrams.is_empty() { + return self.modified.keys().copied().collect(); + } + self.modified + .iter() + .filter_map(|(&file_idx, bigrams)| { + pattern_bigrams + .iter() + .all(|pb| bigrams.contains(pb)) + .then_some(file_idx) + }) + .collect() + } + + /// Number of base files this overlay was created for. + pub(crate) fn base_file_count(&self) -> usize { + self.base_file_count + } + + /// Get the tombstone bitset for clearing base candidates. + pub(crate) fn tombstones(&self) -> &[u64] { + &self.tombstones + } + + /// Get all modified file indices (for conservative overlay merging when + /// we can't extract precise bigrams, e.g. regex patterns). + pub(crate) fn modified_indices(&self) -> Vec { + self.modified.keys().copied().collect() + } +} + +const BIGRAM_CHUNK_FILES: usize = 4 * 64; + +/// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add +/// little filtering power but ~25-30% of index memory, so we drop +/// anything appearing in < 12 % of populated files. +const SKIP_INDEX_MIN_DENSITY_PCT: u32 = 12; + +thread_local! { + /// Reusable read buffer that is allocated per thread and used for reading files + static READ_BUF: std::cell::RefCell> = + std::cell::RefCell::new(vec![0u8; MAX_INDEXABLE_FILE_SIZE].into_boxed_slice()); +} + +/// Reads bigram chunk, we *SHOULD NOT* use mmap cache here because bigram is built off-lock +/// if the watcher thread tries to invalidate mmap during the borrow from it - UAB or segfaut +/// +/// mmap should only be used by the locked version of grep which absolutely minimizes any riscs +#[inline] +fn read_bigram_chunk<'a>( + file: &FileItem, + base_fd: libc::c_int, + base_path: &std::path::Path, + arena: crate::simd_path::ArenaPtr, + buf: &'a mut [u8], + path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE], +) -> Option<&'a [u8]> { + let want = (file.size as usize).min(MAX_INDEXABLE_FILE_SIZE); + let filled = file.read_trimmed_into_buf(base_fd, base_path, arena, path_buf, &mut buf[..want]); + if filled == 0 { + return None; + } + + let data = &buf[..filled]; + + Some(data) +} + +#[tracing::instrument(skip_all, name = "Building Bigram Index", level = tracing::Level::DEBUG)] +pub(crate) fn build_bigram_index( + files: &[crate::types::FileItem], + base_path: &std::path::Path, + arena: crate::simd_path::ArenaPtr, +) -> BigramFilter { + let builder = BigramIndexBuilder::new(files.len()); + let skip_builder = BigramIndexBuilder::new(files.len()); + + #[cfg(unix)] + let base_fd: libc::c_int = open_base_dir_fd(base_path); + #[cfg(not(unix))] + let base_fd: i32 = -1; + + // Always reads each file into the thread-local READ_BUF — never aliases the + // persistent mmap cache. See `read_bigram_chunk` for the rationale: this + // pass runs detached on the background pool without holding the picker + // read lock, so a watcher event mutating a `FileItem` would race any + // borrow we took from a cached `Mmap`. + crate::parallelism::BACKGROUND_THREAD_POOL.install(|| { + files + .par_chunks(BIGRAM_CHUNK_FILES) + .enumerate() + .for_each(|(chunk_idx, chunk)| { + let base_idx = chunk_idx * BIGRAM_CHUNK_FILES; + for (offset, file) in chunk.iter().enumerate() { + let file_idx = base_idx + offset; + + if file.is_binary() || file.size == 0 { + return; + } + + READ_BUF.with(|read_cell| { + let mut buf = read_cell.borrow_mut(); + let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + + if let Some(content) = read_bigram_chunk( + file, + base_fd, + base_path, + arena, + &mut buf[..], + &mut path_buf, + ) { + // we have to manually ensure that every byte is a valid text byte to + // perform this we have to scan every file, first 512 bytes is not enough + // so basically we rely on the fact that first 2MB will always contain + // an invalid text sequence if this is not a binary file. + // + // Need to find a better way to do this. + file.set_binary(crate::types::detect_binary_content(content)); + + builder.add_file_content(&skip_builder, file_idx, content); + } + }); + } + }); + }); + + #[cfg(unix)] + if base_fd >= 0 { + unsafe { libc::close(base_fd) }; + } + + let mut index = builder.compress(None); + let skip_index = skip_builder.compress(Some(SKIP_INDEX_MIN_DENSITY_PCT)); + index.set_skip_index(skip_index); + + // in progress bigram walk + rust's ignore crate allocates shit ton of garbage memory + // all custom allocators would think this is available resource while we do not allocate + // after the sync, so it's very important to let the unused memory go back to the OS + crate::file_picker::hint_allocator_collect(); + + index +} + +#[tracing::instrument(skip_all, name = "Sniffing Large Files Binary", level = tracing::Level::DEBUG)] +pub(crate) fn sniff_binary_for_non_indexable( + files: &[FileItem], + base_path: &std::path::Path, + arena: crate::simd_path::ArenaPtr, + cancelled: &std::sync::atomic::AtomicBool, +) { + // Non-indexable files are few in a typical repo, so a serial pass with a + // single reused chunk buffer beats spinning up the thread pool. + let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + let mut chunk = vec![0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE]; + use std::sync::atomic::Ordering; + + for (i, file) in files.iter().enumerate() { + // check every 256 files to avoid useless work + if (i & 0xFF) == 0 && cancelled.load(Ordering::Acquire) { + return; + } + + // check only the files that we are able to grep + if file.size == 0 || file.size > constants::MAX_FFFILE_SIZE { + continue; + } + + let abs = file.write_absolute_path(arena, base_path, &mut path_buf); + file.detect_binary_per_byte(abs, &mut chunk); + } +} + +/// Open the base directory for the `openat` fast path. Returns `-1` on +/// failure — callers interpret a negative fd as "fall back to absolute +/// paths". +#[cfg(unix)] +fn open_base_dir_fd(base_path: &std::path::Path) -> libc::c_int { + use std::os::unix::ffi::OsStrExt; + let mut cstr = [0u8; crate::simd_path::PATH_BUF_SIZE]; + let bytes = base_path.as_os_str().as_bytes(); + if bytes.len() >= cstr.len() { + return -1; + } + cstr[..bytes.len()].copy_from_slice(bytes); + // SAFETY: `cstr` is NUL-terminated by construction (zero-initialised, + // and we only filled up to `bytes.len() < cstr.len()`). + unsafe { + libc::open( + cstr.as_ptr() as *const std::os::raw::c_char, + libc::O_RDONLY | libc::O_DIRECTORY, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a key the same way `add_file_content` does: two printable-ASCII + /// bytes, lowercased, packed as `(hi << 8) | lo`. + fn key(a: u8, b: u8) -> u16 { + ((a.to_ascii_lowercase() as u16) << 8) | b.to_ascii_lowercase() as u16 + } + + /// Return the sorted list of (consec, skip) bigram keys that should appear + /// for `content`. Used as the reference implementation. + fn expected_bigrams(content: &[u8]) -> (Vec, Vec) { + let mut consec: std::collections::BTreeSet = Default::default(); + let mut skip: std::collections::BTreeSet = Default::default(); + let printable = |b: u8| (32..=126).contains(&b); + for i in 1..content.len() { + let a = content[i - 1]; + let b = content[i]; + if printable(a) && printable(b) { + consec.insert(key(a, b)); + } + if i >= 2 { + let a = content[i - 2]; + let b = content[i]; + if printable(a) && printable(b) { + skip.insert(key(a, b)); + } + } + } + (consec.into_iter().collect(), skip.into_iter().collect()) + } + + /// Query: does the builder record file 0 as having this bigram set? + fn builder_has_key_for_file_0(b: &BigramIndexBuilder, k: u16) -> bool { + let col = b.lookup[k as usize].load(Ordering::Relaxed); + if col == NO_COLUMN { + return false; + } + b.column_bitset(col)[0] & 1 != 0 + } + + fn run_and_compare(content: &[u8]) { + let consec = BigramIndexBuilder::new(1); + let skip = BigramIndexBuilder::new(1); + consec.add_file_content(&skip, 0, content); + + let (expected_consec, expected_skip) = expected_bigrams(content); + + // Every expected bigram must be recorded. + for k in &expected_consec { + assert!( + builder_has_key_for_file_0(&consec, *k), + "consec bigram 0x{k:04x} missing for content {content:?}", + ); + } + for k in &expected_skip { + assert!( + builder_has_key_for_file_0(&skip, *k), + "skip bigram 0x{k:04x} missing for content {content:?}", + ); + } + + // No unexpected bigrams — iterate lookup for set columns. + for k in 0u32..=0xFFFF { + let recorded_consec = builder_has_key_for_file_0(&consec, k as u16); + let recorded_skip = builder_has_key_for_file_0(&skip, k as u16); + if recorded_consec { + assert!( + expected_consec.contains(&(k as u16)), + "unexpected consec bigram 0x{k:04x} in content {content:?}", + ); + } + if recorded_skip { + assert!( + expected_skip.contains(&(k as u16)), + "unexpected skip bigram 0x{k:04x} in content {content:?}", + ); + } + } + } + + #[test] + fn add_file_empty_is_noop() { + let consec = BigramIndexBuilder::new(1); + let skip = BigramIndexBuilder::new(1); + consec.add_file_content(&skip, 0, b""); + assert_eq!(consec.columns_used(), 0); + assert_eq!(skip.columns_used(), 0); + // populated counter not incremented for empty input + assert_eq!(consec.populated.load(Ordering::Relaxed), 0); + } + + #[test] + fn add_file_single_byte_is_noop() { + let consec = BigramIndexBuilder::new(1); + let skip = BigramIndexBuilder::new(1); + consec.add_file_content(&skip, 0, b"a"); + assert_eq!(consec.columns_used(), 0); + assert_eq!(skip.columns_used(), 0); + } + + #[test] + fn add_file_two_bytes_consec_only() { + // With exactly 2 bytes there's no skip bigram (needs i >= 2 in the loop). + run_and_compare(b"ab"); + } + + #[test] + fn add_file_three_bytes_has_skip() { + // "abc" -> consec {"ab", "bc"}, skip {"ac"} + run_and_compare(b"abc"); + } + + #[test] + fn add_file_ascii_words() { + run_and_compare(b"hello world"); + run_and_compare(b"the quick brown fox jumps over the lazy dog"); + run_and_compare(b"fn main() { println!(\"hi\"); }"); + } + + #[test] + fn add_file_case_is_lowered() { + // Uppercase should be lowercased before keying, so "AB" == "ab". + let upper = BigramIndexBuilder::new(1); + let upper_skip = BigramIndexBuilder::new(1); + upper.add_file_content(&upper_skip, 0, b"ABC"); + + let lower = BigramIndexBuilder::new(1); + let lower_skip = BigramIndexBuilder::new(1); + lower.add_file_content(&lower_skip, 0, b"abc"); + + // Both should have identical bigram keys. + for k in 0u32..=0xFFFF { + let u = builder_has_key_for_file_0(&upper, k as u16); + let l = builder_has_key_for_file_0(&lower, k as u16); + assert_eq!(u, l, "consec 0x{k:04x}: upper={u} lower={l}"); + let u = builder_has_key_for_file_0(&upper_skip, k as u16); + let l = builder_has_key_for_file_0(&lower_skip, k as u16); + assert_eq!(u, l, "skip 0x{k:04x}: upper={u} lower={l}"); + } + } + + #[test] + fn add_file_rejects_non_printable() { + // Bigrams where either byte is outside 32..=126 are rejected. But + // the skip-1 bigram can still connect two printable bytes across a + // non-printable one: for "\0a\0b", consec sees no valid pair but + // skip sees (a,b) at i=3. Use the reference implementation. + run_and_compare(b"\0a\0b"); + + // All-zero input: truly nothing recorded. + let consec = BigramIndexBuilder::new(1); + let skip = BigramIndexBuilder::new(1); + consec.add_file_content(&skip, 0, b"\0\0\0\0"); + assert_eq!(consec.columns_used(), 0); + assert_eq!(skip.columns_used(), 0); + } + + #[test] + fn add_file_mixed_printable_and_control() { + // "a\tb\nc d" — \t (9) and \n (10) are below 32. Consec: + // (a, \t) x, (\t, b) x, (b, \n) x, (\n, c) x, (c, ' ') ok, (' ', d) ok + // Skip (i-2, i): + // (a, b) ok, (\t, \n) x, (b, c) ok, (\n, ' ') x, (c, d) ok + run_and_compare(b"a\tb\nc d"); + } + + #[test] + fn add_file_repeats_are_deduped() { + // "ababab" has many repeats of "ab", "ba" — each unique bigram should + // be recorded exactly once (the stack-local `seen_*` dedup works). + run_and_compare(b"ababababab"); + } + + #[test] + fn add_file_tombstone_separation() { + // Two separate files share no bits; file 1's content doesn't bleed + // into file 0's row and vice-versa. + let consec = BigramIndexBuilder::new(2); + let skip = BigramIndexBuilder::new(2); + consec.add_file_content(&skip, 0, b"xy"); + consec.add_file_content(&skip, 1, b"zw"); + + let key_xy = key(b'x', b'y'); + let key_zw = key(b'z', b'w'); + + // file 0 has "xy" but not "zw" + let col_xy = consec.lookup[key_xy as usize].load(Ordering::Relaxed); + let col_zw = consec.lookup[key_zw as usize].load(Ordering::Relaxed); + let bitset_xy = consec.column_bitset(col_xy)[0]; + let bitset_zw = consec.column_bitset(col_zw)[0]; + assert_eq!(bitset_xy & 0b01, 0b01, "file 0 should have xy"); + assert_eq!(bitset_zw & 0b01, 0, "file 0 should NOT have zw"); + assert_eq!(bitset_xy & 0b10, 0, "file 1 should NOT have xy"); + assert_eq!(bitset_zw & 0b10, 0b10, "file 1 should have zw"); + } + + #[test] + fn add_file_long_content() { + // Stress test: ~8 KB of printable ASCII. Should complete without + // overflowing any stack-local bitset and produce the full set. + let mut buf = Vec::with_capacity(8192); + for i in 0..8192 { + buf.push(32u8 + ((i * 7) % 95) as u8); // cycle through printable range + } + run_and_compare(&buf); + } + + #[test] + fn add_file_simd_and_scalar_agree() { + // Cross-check: both code paths (scalar <128 bytes, SIMD ≥128) must + // produce identical bigram sets for content that straddles the + // threshold. Mix printable ASCII with some non-printable bytes and + // repeats so the non-printable branch in the SIMD path exercises. + let mut mixed = Vec::with_capacity(256); + for i in 0..256usize { + mixed.push(match i % 9 { + 0 => 0, // NUL + 1 => 0x7F, // DEL (just above 126) + 2 => b'\n', // below 32 + _ => 32 + ((i * 13) % 95) as u8, + }); + } + + run_and_compare(&mixed[..127]); // scalar path + run_and_compare(&mixed); // SIMD path (256 bytes) + run_and_compare(&mixed[..192]); // SIMD path with scalar tail + } + + #[test] + fn add_file_long_short_paths_agree() { + // Same mixed content checked just below, at, and above + // LONG_CONTENT_MIN_LEN so both add_short_content and add_long_content + // are validated against the reference implementation. + let mut mixed = Vec::with_capacity(LONG_CONTENT_MIN_LEN * 2); + for i in 0..LONG_CONTENT_MIN_LEN * 2 { + mixed.push(match i % 11 { + 0 => 0, + 1 => 0x7F, + 2 => b'\n', + _ => 32 + ((i * 31) % 95) as u8, + }); + } + run_and_compare(&mixed[..LONG_CONTENT_MIN_LEN - 1]); + run_and_compare(&mixed[..LONG_CONTENT_MIN_LEN]); + run_and_compare(&mixed); + } + + #[test] + fn add_file_respects_file_count_boundary() { + // file_count=100, file_idx=63 (last bit in word 0) and file_idx=64 + // (first bit in word 1). Make sure the word_idx math is right. + let consec = BigramIndexBuilder::new(100); + let skip = BigramIndexBuilder::new(100); + consec.add_file_content(&skip, 63, b"ab"); + consec.add_file_content(&skip, 64, b"cd"); + + let kab = key(b'a', b'b'); + let kcd = key(b'c', b'd'); + let col_ab = consec.lookup[kab as usize].load(Ordering::Relaxed); + let col_cd = consec.lookup[kcd as usize].load(Ordering::Relaxed); + + let ab_bitset = consec.column_bitset(col_ab); + let cd_bitset = consec.column_bitset(col_cd); + // ab in word 0, bit 63 + assert_eq!(ab_bitset[0], 1u64 << 63); + assert_eq!(ab_bitset[1], 0); + // cd in word 1, bit 0 + assert_eq!(cd_bitset[0], 0); + assert_eq!(cd_bitset[1], 1); + } +} diff --git a/crates/fff-core/src/bigram_query.rs b/crates/fff-core/src/bigram_query.rs new file mode 100644 index 0000000..fd7c57f --- /dev/null +++ b/crates/fff-core/src/bigram_query.rs @@ -0,0 +1,998 @@ +use crate::bigram_filter::BigramFilter; +use regex_syntax::hir::{Class, Hir, HirKind}; +use smallvec::SmallVec; +use std::borrow::Cow; + +const MAX_CLASS_EXPAND: usize = 16; + +// stack inlined array padded with 0 and tracked length +#[derive(Clone, Copy, PartialEq, Eq)] +struct InlineArray { + bytes: [u8; MAX_CLASS_EXPAND], + len: usize, +} + +impl InlineArray { + const fn new() -> Self { + Self { + bytes: [0; MAX_CLASS_EXPAND], + len: 0, + } + } + + fn from_byte(b: u8) -> Self { + let mut set = Self::new(); + set.push(b); + set + } + + /// Append a byte; no-op if already full (callers guard against this). + fn push(&mut self, b: u8) { + if self.len < MAX_CLASS_EXPAND { + self.bytes[self.len] = b; + self.len += 1; + } + } +} + +impl std::ops::Deref for InlineArray { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + +#[inline] +fn consec_key(a: u8, b: u8) -> Option { + let al = a.to_ascii_lowercase(); + let bl = b.to_ascii_lowercase(); + if (32..=126).contains(&al) && (32..=126).contains(&bl) { + Some((al as u16) << 8 | bl as u16) + } else { + None + } +} + +#[derive(Debug, Clone)] +pub enum BigramQuery { + Any, + /// A consecutive bigram key to look up in the main index. + Consec(u16), + /// A skip-1 bigram key to look up in the skip sub-index. + Skip1(u16), + /// All children must match (intersect posting lists). + And(Vec), + /// At least one child must match (union posting lists). + Or(Vec), +} + +/// SIMD-friendly bitwise OR of two equal-length bitsets. +#[inline] +fn bitset_or(a: &mut [u64], b: &[u64]) { + a.iter_mut().zip(b.iter()).for_each(|(x, y)| *x |= *y); +} + +/// SIMD-friendly bitwise AND of two equal-length bitsets. +#[inline] +fn bitset_and(a: &mut [u64], b: &[u64]) { + a.iter_mut().zip(b.iter()).for_each(|(x, y)| *x &= *y); +} + +impl BigramQuery { + pub fn is_any(&self) -> bool { + matches!(self, BigramQuery::Any) + } + + pub(crate) fn evaluate(&self, index: &BigramFilter) -> Option> { + self.evaluate_cow(index).map(Cow::into_owned) + } + + fn evaluate_cow<'a>(&self, index: &'a BigramFilter) -> Option> { + match self { + BigramQuery::Any => None, + + BigramQuery::Consec(key) => { + let col = index.lookup()[*key as usize]; + if col == u16::MAX { + return None; + } + let words = index.words(); + let offset = col as usize * words; + let data = index.dense_data(); + if offset + words > data.len() { + return None; + } + Some(Cow::Borrowed(&data[offset..offset + words])) + } + + BigramQuery::Skip1(key) => { + let skip = index.skip_index()?; + let col = skip.lookup()[*key as usize]; + if col == u16::MAX { + return None; + } + let words = skip.words(); + let offset = col as usize * words; + let data = skip.dense_data(); + if offset + words > data.len() { + return None; + } + Some(Cow::Borrowed(&data[offset..offset + words])) + } + + BigramQuery::And(children) => { + let mut result: Option> = None; + for child in children { + if let Some(child_bits) = child.evaluate_cow(index) { + result = Some(match result { + None => child_bits.into_owned(), + Some(mut r) => { + bitset_and(&mut r, &child_bits); + r + } + }); + } + } + result.map(Cow::Owned) + } + + BigramQuery::Or(children) => { + if children.is_empty() { + return None; + } + let mut result: Option> = None; + for child in children { + // Any branch can't be filtered -> whole OR can't be filtered + let child_bits = child.evaluate_cow(index)?; + result = Some(match result { + None => child_bits.into_owned(), + Some(mut r) => { + bitset_or(&mut r, &child_bits); + r + } + }); + } + result.map(Cow::Owned) + } + } + } +} + +struct HirInfo { + query: BigramQuery, + first: Option, + last: Option, + can_be_empty: bool, +} + +impl HirInfo { + fn empty() -> Self { + Self { + query: BigramQuery::Any, + first: None, + last: None, + can_be_empty: true, + } + } +} + +/// Prefilter fuzzy query. The algorithm is the following: +/// we allow max_typos = min(len/3,2) every typo destroys at most 2 consecutive bigrams +/// So out of N bigrams at least N - 2 * max_typos have to present in the matching fil +pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQuery { + let lower: Vec = query.bytes().map(|b| b.to_ascii_lowercase()).collect(); + + if lower.len() < 2 { + return BigramQuery::Any; + } + + let max_typos = (lower.len() / 3).min(2); + + // Extract all consecutive bigram keys. + let bigram_keys: Vec = lower + .windows(2) + .filter_map(|w| consec_key(w[0], w[1])) + .collect(); + + if bigram_keys.is_empty() { + return BigramQuery::Any; + } + + // the simplest case, just check that every bigram is present either consec or not + if max_typos == 0 { + return simplify_and( + bigram_keys + .iter() + .map(|&k| BigramQuery::Consec(k)) + .collect(), + ); + } + + // Pick evenly-spaced probe bigrams. + let n = num_probes.min(bigram_keys.len()); + if n <= max_typos { + // Too few probes to require anything useful. + return simplify_or( + bigram_keys + .iter() + .map(|&k| BigramQuery::Consec(k)) + .collect(), + ); + } + + let probes: Vec = if n == bigram_keys.len() { + bigram_keys + } else { + (0..n) + .map(|i| { + let idx = i * (bigram_keys.len() - 1) / (n - 1); + bigram_keys[idx] + }) + .collect() + }; + + let required = n - max_typos; + + // If required == n, just AND all probes. + if required >= n { + return simplify_and(probes.iter().map(|&k| BigramQuery::Consec(k)).collect()); + } + + // Generate all C(n, required) subsets as OR(AND(subset), ...) + let mut branches = Vec::new(); + let mut combo = vec![0u16; required]; + combine(&probes, required, 0, 0, &mut combo, &mut branches); + + simplify_or(branches) +} + +/// Build C(n, k) combination branches in-place on a fixed-size slice. +fn combine( + items: &[u16], + k: usize, + start: usize, + depth: usize, + combo: &mut [u16], + branches: &mut Vec, +) { + if depth == k { + branches.push(simplify_and( + combo.iter().map(|&key| BigramQuery::Consec(key)).collect(), + )); + return; + } + let remaining = k - depth; + for i in start..=items.len() - remaining { + combo[depth] = items[i]; + combine(items, k, i + 1, depth + 1, combo, branches); + } +} + +pub(crate) fn regex_to_bigram_query(pattern: &str) -> BigramQuery { + let mut parser = regex_syntax::ParserBuilder::new() + .unicode(false) + .utf8(false) + .build(); + + let hir = match parser.parse(pattern) { + Ok(h) => h, + Err(_) => return BigramQuery::Any, + }; + + decompose(&hir).query +} + +fn decompose(hir: &Hir) -> HirInfo { + let can_be_empty = hir.properties().minimum_len().is_none_or(|n| n == 0); + + match hir.kind() { + HirKind::Empty => HirInfo::empty(), + + HirKind::Literal(lit) => decompose_literal(lit.0.as_ref()), + + HirKind::Class(class) => { + let bytes = expand_class(class); + match bytes { + Some(b) if !b.is_empty() => HirInfo { + query: BigramQuery::Any, + first: Some(b), + last: Some(b), + can_be_empty, + }, + _ => HirInfo { + query: BigramQuery::Any, + first: None, + last: None, + can_be_empty, + }, + } + } + + HirKind::Look(_) => HirInfo::empty(), + + HirKind::Repetition(rep) => { + let inner = decompose(&rep.sub); + if rep.min == 0 { + HirInfo { + query: BigramQuery::Any, + first: inner.first, + last: inner.last, + can_be_empty: true, + } + } else { + // min >= 1: inner bigrams guaranteed + let mut qs = Vec::new(); + if !inner.query.is_any() { + qs.push(inner.query.clone()); + } + // min >= 2: cross-boundary between consecutive occurrences + if rep.min >= 2 { + push_cross_consec(&mut qs, inner.last.as_deref(), inner.first.as_deref()); + } + HirInfo { + query: simplify_and(qs), + first: inner.first, + last: inner.last, + can_be_empty, + } + } + } + + HirKind::Capture(cap) => decompose(&cap.sub), + + HirKind::Concat(parts) => decompose_concat(parts), + + HirKind::Alternation(alts) => decompose_alternation(alts), + } +} + +/// Extract bigrams from a literal byte sequence. +fn decompose_literal(bytes: &[u8]) -> HirInfo { + if bytes.is_empty() { + return HirInfo::empty(); + } + + let lower: SmallVec<[u8; 64]> = bytes.iter().map(|b| b.to_ascii_lowercase()).collect(); + + if lower.len() == 1 { + let b = lower[0]; + let first = if (32..=126).contains(&b) { + Some(InlineArray::from_byte(b)) + } else { + None + }; + return HirInfo { + query: BigramQuery::Any, + first, + last: first, + can_be_empty: false, + }; + } + + let mut qs: Vec = Vec::new(); + + // Consecutive bigrams + for w in lower.windows(2) { + if let Some(k) = consec_key(w[0], w[1]) { + qs.push(BigramQuery::Consec(k)); + } + } + + // Skip-1 bigrams from the literal itself + if lower.len() >= 3 { + for i in 0..lower.len() - 2 { + if let Some(k) = consec_key(lower[i], lower[i + 2]) { + qs.push(BigramQuery::Skip1(k)); + } + } + } + + let first_byte = lower[0]; + let last_byte = *lower.last().unwrap(); + + HirInfo { + query: simplify_and(qs), + first: if (32..=126).contains(&first_byte) { + Some(InlineArray::from_byte(first_byte)) + } else { + None + }, + last: if (32..=126).contains(&last_byte) { + Some(InlineArray::from_byte(last_byte)) + } else { + None + }, + can_be_empty: false, + } +} + +fn decompose_concat(parts: &[Hir]) -> HirInfo { + if parts.is_empty() { + return HirInfo::empty(); + } + + let infos: Vec = parts.iter().map(decompose).collect(); + let mut qs: Vec = Vec::new(); + + for info in &infos { + if !info.query.is_any() { + qs.push(info.query.clone()); + } + } + + // Dense cross-boundary between adjacent mandatory parts + for pair in infos.windows(2) { + if !pair[0].can_be_empty && !pair[1].can_be_empty { + push_cross_consec(&mut qs, pair[0].last.as_deref(), pair[1].first.as_deref()); + } + } + + // Sparse-1 cross-boundary: across a single 1 byte wide middle part + if parts.len() >= 3 { + for i in 0..parts.len() - 2 { + let left = &infos[i]; + let mid = &parts[i + 1]; + let right = &infos[i + 2]; + + let min_len = mid.properties().minimum_len(); + let max_len = mid.properties().maximum_len(); + let is_1byte = min_len == Some(1) && max_len == Some(1); + + if is_1byte && !left.can_be_empty && !right.can_be_empty { + push_cross_skip1(&mut qs, left.last.as_deref(), right.first.as_deref()); + } + } + } + + let first = collect_first(&infos); + let last = collect_last(&infos); + let can_be_empty = infos.iter().all(|i| i.can_be_empty); + + HirInfo { + query: simplify_and(qs), + first, + last, + can_be_empty, + } +} + +fn decompose_alternation(alts: &[Hir]) -> HirInfo { + if alts.is_empty() { + return HirInfo::empty(); + } + + let infos: Vec = alts.iter().map(decompose).collect(); + let query = simplify_or(infos.iter().map(|i| i.query.clone()).collect()); + let first = merge_byte_sets(infos.iter().map(|i| &i.first)); + let last = merge_byte_sets(infos.iter().map(|i| &i.last)); + let can_be_empty = infos.iter().any(|i| i.can_be_empty); + + HirInfo { + query, + first, + last, + can_be_empty, + } +} + +fn expand_class(class: &Class) -> Option { + let mut bytes = InlineArray::new(); + match class { + Class::Bytes(bc) => { + for range in bc.ranges() { + let count = (range.end() as usize) - (range.start() as usize) + 1; + if bytes.len() + count > MAX_CLASS_EXPAND { + return None; + } + + for b in range.start()..=range.end() { + if (32..=126).contains(&b) { + let lower = b.to_ascii_lowercase(); + if !bytes.contains(&lower) { + bytes.push(lower); + } + } + } + } + } + Class::Unicode(uc) => { + for range in uc.ranges() { + let start = range.start() as u32; + let end = range.end() as u32; + if start > 127 { + continue; + } + let ascii_end = end.min(126) as u8; + let ascii_start = start.max(32) as u8; + if ascii_start > ascii_end { + continue; + } + let count = (ascii_end - ascii_start) as usize + 1; + if bytes.len() + count > MAX_CLASS_EXPAND { + return None; + } + for b in ascii_start..=ascii_end { + let lower = b.to_ascii_lowercase(); + if !bytes.contains(&lower) { + bytes.push(lower); + } + } + } + } + } + if bytes.is_empty() { None } else { Some(bytes) } +} + +/// Push consecutive cross-product bigrams into `qs`. +fn push_cross_consec(qs: &mut Vec, last: Option<&[u8]>, first: Option<&[u8]>) { + if let Some(q) = cross_product(last, first, false) { + qs.push(q); + } +} + +/// Push skip-1 cross-product bigrams into `qs`. +fn push_cross_skip1(qs: &mut Vec, last: Option<&[u8]>, first: Option<&[u8]>) { + if let Some(q) = cross_product(last, first, true) { + qs.push(q); + } +} + +fn cross_product(last: Option<&[u8]>, first: Option<&[u8]>, skip: bool) -> Option { + let last = last?; + let first = first?; + let n = last.len() * first.len(); + if n == 0 || n > MAX_CLASS_EXPAND * MAX_CLASS_EXPAND { + return None; + } + + let mut bigrams: Vec = Vec::with_capacity(n); + for &l in last { + for &f in first { + if let Some(k) = consec_key(l, f) { + let node = if skip { + BigramQuery::Skip1(k) + } else { + BigramQuery::Consec(k) + }; + bigrams.push(node); + } + } + } + + match bigrams.len() { + 0 => None, + 1 => Some(bigrams.into_iter().next().unwrap()), + _ => Some(simplify_or(bigrams)), + } +} + +fn collect_first(infos: &[HirInfo]) -> Option { + let mut result = InlineArray::new(); + for info in infos { + if let Some(ref bytes) = info.first { + for &b in bytes.iter() { + if !result.contains(&b) { + if result.len() >= MAX_CLASS_EXPAND { + return None; + } + result.push(b); + } + } + } else if !info.can_be_empty { + return None; + } + if !info.can_be_empty { + break; + } + } + if result.is_empty() { + None + } else { + Some(result) + } +} + +fn collect_last(infos: &[HirInfo]) -> Option { + let mut result = InlineArray::new(); + for info in infos.iter().rev() { + if let Some(ref bytes) = info.last { + for &b in bytes.iter() { + if !result.contains(&b) { + if result.len() >= MAX_CLASS_EXPAND { + return None; + } + result.push(b); + } + } + } else if !info.can_be_empty { + return None; + } + if !info.can_be_empty { + break; + } + } + if result.is_empty() { + None + } else { + Some(result) + } +} + +fn merge_byte_sets<'a>(iter: impl Iterator>) -> Option { + let mut result = InlineArray::new(); + for opt in iter { + let bytes = opt.as_ref()?; + + for &b in bytes.iter() { + if !result.contains(&b) { + if result.len() >= MAX_CLASS_EXPAND { + return None; + } + result.push(b); + } + } + } + if result.is_empty() { + None + } else { + Some(result) + } +} + +fn simplify_and(children: Vec) -> BigramQuery { + let mut flat: Vec = Vec::new(); + for child in children { + match child { + BigramQuery::Any => {} + BigramQuery::And(inner) => flat.extend(inner), + other => flat.push(other), + } + } + match flat.len() { + 0 => BigramQuery::Any, + 1 => flat.into_iter().next().unwrap(), + _ => BigramQuery::And(flat), + } +} + +fn simplify_or(children: Vec) -> BigramQuery { + if children.iter().any(|c| c.is_any()) { + return BigramQuery::Any; + } + let mut flat: Vec = Vec::new(); + for child in children { + match child { + BigramQuery::Or(inner) => flat.extend(inner), + other => flat.push(other), + } + } + match flat.len() { + 0 => BigramQuery::Any, + 1 => flat.into_iter().next().unwrap(), + _ => BigramQuery::Or(flat), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bigram_filter::BigramIndexBuilder; + + /// Build a tiny index from the given file contents for testing. + fn build_test_index(files: &[&[u8]]) -> BigramFilter { + let n = files.len(); + let consec_builder = BigramIndexBuilder::new(n); + let skip_builder = BigramIndexBuilder::new(n); + for (i, content) in files.iter().enumerate() { + consec_builder.add_file_content(&skip_builder, i, content); + } + let mut idx = consec_builder.compress(Some(0)); + idx.set_skip_index(skip_builder.compress(Some(0))); + idx + } + + #[test] + fn literal_pattern() { + let idx = build_test_index(&[ + b"hello world", // 0: contains "hello" + b"goodbye world", // 1: no "hello" + b"say hello there", // 2: contains "hello" + ]); + + let q = regex_to_bigram_query("hello"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(!BigramFilter::is_candidate(&candidates, 1)); + assert!(BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn alternation() { + let idx = build_test_index(&[ + b"has foo in it", // 0 + b"has bar in it", // 1 + b"has xyz in it", // 2 + ]); + + let q = regex_to_bigram_query("foo|bar"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(BigramFilter::is_candidate(&candidates, 1)); + // xyz doesn't contain foo or bar bigrams + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn wildcard_concat() { + let idx = build_test_index(&[ + b"foo something bar", // 0 + b"foo only", // 1: has foo but not bar + b"only bar", // 2: has bar but not foo + ]); + + let q = regex_to_bigram_query("foo.*bar"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + // file 1 and 2 should be filtered (missing bigrams from "bar" / "foo") + assert!(!BigramFilter::is_candidate(&candidates, 1)); + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn sparse1_across_dot() { + // "a.b" should produce a skip-1 bigram (a,b) + let idx = build_test_index(&[ + b"axb", // 0: has sparse-1 (a,b) + b"ayb", // 1: has sparse-1 (a,b) + b"xyz", // 2: no (a,b) at all + ]); + + let q = regex_to_bigram_query("a.b"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(BigramFilter::is_candidate(&candidates, 1)); + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn sparse1_across_digit() { + // "foo\dbar" -> sparse-1 (o,b) across \d + let idx = build_test_index(&[ + b"foo3bar baz", // 0: has all bigrams + b"foobar baz", // 1: has consecutive (o,b) but pattern needs sparse-1 + b"xyz only", // 2: no relevant bigrams + ]); + + let q = regex_to_bigram_query(r"foo\dbar"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + // file 1 may or may not match depending on what bigrams are in the index + // (it has all the literal bigrams and also o,b as both consec and skip-1) + // The important thing is file 2 is excluded: + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn pure_wildcard_is_any() { + let q = regex_to_bigram_query(".*"); + assert!(q.is_any()); + } + + #[test] + fn single_char_is_any() { + let q = regex_to_bigram_query("a"); + assert!(q.is_any()); + } + + #[test] + fn invalid_regex_is_any() { + let q = regex_to_bigram_query("[invalid"); + assert!(q.is_any()); + } + + #[test] + fn optional_group_excluded() { + let q = regex_to_bigram_query("foo(bar)?baz"); + assert!(!q.is_any()); + + let idx = build_test_index(&[ + b"foobaz content", // 0: has foo+baz bigrams (bar absent) + b"foobarbaz content", // 1: has everything + b"xyz only", // 2: nothing + ]); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(BigramFilter::is_candidate(&candidates, 1)); + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn repetition_min2_cross_boundary() { + // (ab){2,} -> bigram "ab" + cross-boundary "b","a" + let q = regex_to_bigram_query("(ab){2,}"); + assert!(!q.is_any()); + + let idx = build_test_index(&[ + b"ababab", // 0: has "ab" and "b"->"a" + b"abonly", // 1: has "ab" but not "b"->"a" + b"xyz", // 2: nothing + ]); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(!BigramFilter::is_candidate(&candidates, 2)); + } + + #[test] + fn two_dots_no_sparse1() { + let q = regex_to_bigram_query("a..b"); + // Single-char literals with 2 unknown bytes between -> Any + assert!(q.is_any()); + } + + #[test] + fn character_class_cross_boundary() { + // [abc]de -> cross-boundary OR(ad,bd,cd) + bigram de + // All three class variants must appear in the corpus so the OR + // branches are tracked in the index (untracked bigrams make the + // OR conservatively return None, which is correct but untestable). + let idx = build_test_index(&[ + b"ade content", // 0: has ad + b"bde content", // 1: has bd + b"cde content", // 2: has cd + b"xde content", // 3: has de but not ad/bd/cd + ]); + + let q = regex_to_bigram_query("[abc]de"); + assert!(!q.is_any()); + + let candidates = q.evaluate(&idx).unwrap(); + assert!(BigramFilter::is_candidate(&candidates, 0)); + assert!(BigramFilter::is_candidate(&candidates, 1)); + assert!(BigramFilter::is_candidate(&candidates, 2)); + // file 3 doesn't have ad/bd/cd so should be filtered + assert!(!BigramFilter::is_candidate(&candidates, 3)); + } + + fn has_consec(q: &BigramQuery, a: u8, b: u8) -> bool { + let Some(key) = consec_key(a, b) else { + return false; + }; + match q { + BigramQuery::Consec(k) => *k == key, + BigramQuery::And(cs) | BigramQuery::Or(cs) => cs.iter().any(|c| has_consec(c, a, b)), + _ => false, + } + } + + fn has_skip1(q: &BigramQuery, a: u8, b: u8) -> bool { + let Some(key) = consec_key(a, b) else { + return false; + }; + match q { + BigramQuery::Skip1(k) => *k == key, + BigramQuery::And(cs) | BigramQuery::Or(cs) => cs.iter().any(|c| has_skip1(c, a, b)), + _ => false, + } + } + + /// Bigram expectation: `("ab", is_skip1)`. + /// The 2-char str is the byte pair; C = consecutive, S = skip-1. + type Bg = (&'static str, bool); + const C: bool = false; + const S: bool = true; + + /// Top 15+ commonly used regex patterns from + /// https://digitalfortress.tech/tips/top-15-commonly-used-regex/ + /// plus typical grep patterns used by agentic tools. + /// + /// Each entry: `(regex, Option<&[Bg]>)`. + /// - `None` -> pure classes / unsupported syntax, Any is acceptable. + /// - `Some(&[..])` -> must be non-Any, and every listed bigram must appear. + #[test] + fn common_regex_patterns() { + #[rustfmt::skip] + let cases: &[(&str, Option<&[Bg]>)] = &[ + (r"^\d+$", None), // 1. whole numbers + (r"^\d*\.\d+$", None), // 2. decimals + (r"^\d*(\.\d+)?$", None), // 3. whole + decimal + (r"^-?\d*(\.\d+)?$", None), // 4. neg/pos decimal + (r"[-]?[0-9]+[,.]?[0-9]*([/][0-9]+[,.]?[0-9]*)*", None), // 5. fractions + (r"^[a-zA-Z0-9]*$", None), // 6. alphanumeric + (r"^[a-zA-Z0-9 ]*$", None), // 7. alphanum + space + (r"^([a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6})*$", None), // 8. email + (r"^([a-z0-9_\.\+-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})$", None), // 9. email v2 + (r"(?=(.*[0-9]))(?=.*[!@#$%^&*()\[\]{}\-_+=~`|:;<>,./?\x5c])(?=.*[a-z])(?=(.*[A-Z]))(?=(.*)).{8,}", None), // 10. complex pw + (r"(?=(.*[0-9]))((?=.*[A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z]))^.{8,}$", None), // 11. moderate pw + (r"^[a-z0-9_-]{3,16}$", None), // 12. username + (r"(https?://)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", None), // 14. URL optional + (r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", None), // 15. IPv4 + (r"(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))", None), // 16. IPv6 + (r"[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])", None), // 17. date + (r"^(0?[1-9]|1[0-2]):[0-5][0-9]$", None), // 18. time 12h + (r"((1[0-2]|0?[1-9]):([0-5][0-9]) ?([AaPp][Mm]))", None), // 19. time AM/PM + (r"^(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9]$", None), // 20. time 24h + (r"^([0-9]|0[0-9]|1[0-9]|2[0-3]):[0-5][0-9]$", None), // 21. time 24h v2 + (r"(?:[01]\d|2[0123]):(?:[012345]\d):(?:[012345]\d)", None), // 22. time+sec + (r"|<.+[\W]>", None), // 23. HTML tag + (r"\bon\w+=\S+(?=.*>)", None), // 24. inline JS + (r"^[a-z0-9]+(?:-[a-z0-9]+)*$", None), // 25. slug + (r"(\b\w+\b)(?=.*\b\1\b)", None), // 26. dup words + (r"^[\w,\s-]+\.[A-Za-z]{3}$", None), // 27. filename + (r"^[A-PR-WY][1-9]\d\s?\d{4}[1-9]$", None), // 28. HK ID + + // 13. URL with required protocol + (r"https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", Some(&[ + ("ht", C), ("tt", C), ("tp", C), // from "http" + ("ht", S), ("tp", S), // from "http" skip-1 + (":/", C), ("//", C), // from "://" + ])), + + // 29. fn\s+\w+ + (r"fn\s+\w+", Some(&[ + ("fn", C), // from "fn" + ("n ", C), // cross-boundary: 'n' -> \s starts ' ' + ])), + + // 30. use\s+crate:: + (r"use\s+crate::", Some(&[ + ("us", C), ("se", C), ("ue", S), // from "use" + ("cr", C), ("ra", C), ("at", C), // from "crate" + ("te", C), ("::", C), + ("ca", S), ("rt", S), ("ae", S), // "crate" skip-1 + ])), + + // 31. unwrap\(\)|expect\( + (r"unwrap\(\)|expect\(", Some(&[ + ("nw", C), ("wr", C), ("ra", C), // "unwrap(" + ("ap", C), ("p(", C), + ("xp", C), ("pe", C), ("ec", C), // "expect(" + ("ct", C), ("t(", C), + ])), + + // 32. TODO|FIXME|HACK + (r"TODO|FIXME|HACK", Some(&[ + ("to", C), ("od", C), ("do", C), // "TODO" + ("fi", C), ("ix", C), ("xm", C), // "FIXME" + ("me", C), + ("ha", C), ("ac", C), ("ck", C), // "HACK" + ("hc", S), ("ak", S), // "HACK" skip-1 + ])), + ]; + + for (i, &(pattern, expected)) in cases.iter().enumerate() { + let q = regex_to_bigram_query(pattern); + + if let Some(bigrams) = expected { + assert!( + !q.is_any(), + "#{i} {pattern:?}: expected bigrams but got Any" + ); + + for &(pair, skip) in bigrams { + let b = pair.as_bytes(); + debug_assert_eq!(b.len(), 2, "bigram must be 2 chars: {pair:?}"); + let found = if skip { + has_skip1(&q, b[0], b[1]) + } else { + has_consec(&q, b[0], b[1]) + }; + let kind = if skip { "skip-1" } else { "consec" }; + assert!(found, "#{i} {pattern:?}: missing {kind} bigram {pair:?}"); + } + } + } + } +} diff --git a/crates/fff-core/src/constants.rs b/crates/fff-core/src/constants.rs new file mode 100644 index 0000000..95cdc2b --- /dev/null +++ b/crates/fff-core/src/constants.rs @@ -0,0 +1,39 @@ +/// Largest file whose full content fff will touch: the default grep read cap +/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap +/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to +/// this far so nothing grep would read is left unclassified. +pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024; + +/// Upper bound on a file the bigram builder will build, if the file is very large there is a +/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter +pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024; + +/// Total bytes the persistent content mmap cache may hold for a small repo. +pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024; + +/// Files below one page waste the remainder when mmapped, so the cache skips +/// them and falls back to chunked reads. Unused on Windows (no content cache). +#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))] +pub const MMAP_THRESHOLD: u64 = 16 * 1024; +#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))] +pub const MMAP_THRESHOLD: u64 = 4 * 1024; + +/// Capacity reserved for files the watcher discovers after the initial scan; +/// exceeding it forces a full rescan. +pub const MAX_OVERFLOW_FILES: usize = 1024; + +/// Fresh-mmap threshold: files at or above this size get mmapped directly on +/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform. +/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this +/// constant is gated to non-Windows targets to keep `-D unused-imports` happy. +#[cfg(target_os = "macos")] +pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024; +#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] +pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024; + +// we do not support 32kb path limit on windows +#[cfg(target_os = "windows")] +pub const PATH_BUF_SIZE: usize = 4096; + +#[cfg(not(target_os = "windows"))] +pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize; diff --git a/crates/fff-core/src/constraints.rs b/crates/fff-core/src/constraints.rs new file mode 100644 index 0000000..c70e9bb --- /dev/null +++ b/crates/fff-core/src/constraints.rs @@ -0,0 +1,932 @@ +//! Constraint-based prefiltering for search queries. + +use fff_query_parser::{Constraint, GitStatusFilter}; +use smallvec::SmallVec; + +use crate::git::is_modified_status; +use crate::simd_path::ArenaPtr; +use crate::simd_string_utils::memmem::find_case_insensitive_short; + +const PAR_THRESHOLD: usize = 10_000; + +pub(crate) trait Constrainable { + fn write_file_name(&self, arena: ArenaPtr, out: &mut String); + fn git_status(&self) -> Option; + fn write_relative_path(&self, arena: ArenaPtr, out: &mut String); + fn is_overflow(&self) -> bool; +} + +/// Stored/canonical paths use `/`; also accept `\` so a Windows user typing +/// a native separator in a query still matches. +#[inline] +fn is_path_sep(b: u8) -> bool { + b == b'/' || b == b'\\' +} + +#[inline] +fn path_slice_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + a.iter().zip(b).all(|(x, y)| { + if is_path_sep(*x) && is_path_sep(*y) { + true + } else { + x.eq_ignore_ascii_case(y) + } + }) +} + +/// Path ends with suffix at a path-separator boundary (case-insensitive). +#[inline] +pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool { + let path_bytes = path.as_bytes(); + let suffix_bytes = suffix.as_bytes(); + if path_bytes.len() < suffix_bytes.len() { + return false; + } + + let start = path.len() - suffix.len(); + + // Multi-byte UTF-8 may put `start` inside a char. + if !path.is_char_boundary(start) { + return false; + } + + if !path_slice_eq(&path_bytes[start..], suffix_bytes) { + return false; + } + + // Exact or preceded by a separator. Scan backward past any multi-byte + // continuation bytes to find the preceding ASCII byte. + if start == 0 { + return true; + } + let mut i = start; + while i > 0 { + i -= 1; + if path_bytes[i] < 128 { + return is_path_sep(path_bytes[i]); + } + } + false +} + +#[inline] +pub fn file_has_extension(file_name: &str, ext: &str) -> bool { + let name_bytes = file_name.as_bytes(); + let ext_bytes = ext.as_bytes(); + if name_bytes.len() <= ext_bytes.len() + 1 { + return false; + } + let start = name_bytes.len() - ext_bytes.len() - 1; + if start > 0 && !file_name.is_char_boundary(start) { + return false; + } + name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes) +} + +/// Matches multi-segment queries like `libswscale/aarch64`. +#[inline] +pub fn path_contains_segment(path: &str, segment: &str) -> bool { + let path_bytes = path.as_bytes(); + let segment_bytes = segment.as_bytes(); + let segment_len = segment_bytes.len(); + + if path_bytes.len() > segment_len + && is_path_sep(path_bytes[segment_len]) + && path.is_char_boundary(segment_len) + && path_slice_eq(&path_bytes[..segment_len], segment_bytes) + { + return true; + } + + if path_bytes.len() < segment_len + 2 { + return false; + } + + for i in 0..path_bytes.len().saturating_sub(segment_len + 1) { + if is_path_sep(path_bytes[i]) { + let start = i + 1; + let end = start + segment_len; + if end < path_bytes.len() + && is_path_sep(path_bytes[end]) + && path.is_char_boundary(start) + && path.is_char_boundary(end) + && path_slice_eq(&path_bytes[start..end], segment_bytes) + { + return true; + } + } + } + false +} + +/// Returns `None` if no constraints are present, `Some(filtered)` otherwise. +/// +/// Constraint semantics: +/// - All `Extension` constraints OR together (file matches if ANY extension hits). +/// They're split out up front so the per-item loop reads the OR predicate as a +/// single short-circuit check, not as N AND-merged sub-constraints. +/// - Every other constraint kind ANDs (file matches only if ALL hold). They're +/// evaluated in order with short-circuit on first failure. +pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>( + items: &'a [T], + constraints: &[Constraint<'_>], + base_arena: ArenaPtr, + overflow_arena: ArenaPtr, +) -> Option> { + if constraints.is_empty() { + return None; + } + let plan = ConstraintPlan::build(constraints, items, base_arena, overflow_arena); + Some(plan.run(items, base_arena, overflow_arena)) +} + +#[cfg(feature = "zlob")] +type GlobPattern = zlob::ZlobPattern; +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +type GlobPattern = globset::GlobMatcher; + +/// How `Constraint::Glob` is evaluated for each item. +enum GlobStrategy { + /// No Glob constraint present. + None, + /// Pure-glob workload (no Extension filter to reject items first). + /// Batch all paths through zlob/globset once; per-item check is a Vec lookup. + Prepass(Vec>), + /// Mixed workload (Extension filter present). Compile patterns up front, then + /// only run them on items that survive the cheap Extension OR check. + /// `None` slot = compile failure -> never matches; preserves index alignment. + Inline(Vec>), +} + +/// Bundles preprocessed constraints for the per-item evaluator. +pub(crate) struct ConstraintPlan<'q, 'c> { + /// OR semantics — file passes if ANY extension matches. Empty = no ext filter. + extensions: SmallVec<[&'q str; 8]>, + /// AND semantics — file passes only if ALL match. + rest: SmallVec<[&'c Constraint<'q>; 8]>, + glob: GlobStrategy, +} + +pub(crate) struct ConstraintsBuffers { + fname: String, + path: String, +} + +impl ConstraintsBuffers { + pub(crate) fn new() -> Self { + Self { + fname: String::with_capacity(64), + path: String::with_capacity(64), + } + } +} + +impl<'q, 'c> ConstraintPlan<'q, 'c> { + pub(crate) fn build( + constraints: &'c [Constraint<'q>], + items: &[T], + base_arena: ArenaPtr, + overflow_arena: ArenaPtr, + ) -> Self { + let mut extensions = SmallVec::new(); + let mut rest: SmallVec<[&'c Constraint<'q>; 8]> = SmallVec::new(); + for c in constraints { + match c { + Constraint::Extension(ext) => extensions.push(*ext), + _ => rest.push(c), + } + } + let has_pre_filter = !extensions.is_empty() || rest.iter().any(|&c| !is_glob_node(c)); + let glob = build_glob_strategy(&rest, has_pre_filter, items, base_arena, overflow_arena); + + Self { + extensions, + rest, + glob, + } + } + + fn run<'a, T: Constrainable + Sync>( + &self, + items: &'a [T], + base_arean: ArenaPtr, + overflow_arena: ArenaPtr, + ) -> Vec<&'a T> { + if items.len() >= PAR_THRESHOLD { + use rayon::prelude::*; + items + .par_iter() + .enumerate() + .map_init(ConstraintsBuffers::new, |scratch, (i, item)| { + self.matches(item, i, base_arean, overflow_arena, scratch) + .then_some(item) + }) + .flatten() + .collect() + } else { + let mut scratch = ConstraintsBuffers::new(); + items + .iter() + .enumerate() + .filter_map(|(i, item)| { + self.matches(item, i, base_arean, overflow_arena, &mut scratch) + .then_some(item) + }) + .collect() + } + } + + #[inline] + pub(crate) fn matches( + &self, + item: &T, + index: usize, + base_arena: ArenaPtr, + overflow_arena: ArenaPtr, + scratch: &mut ConstraintsBuffers, + ) -> bool { + let arena = if item.is_overflow() { + overflow_arena + } else { + base_arena + }; + + if !self.passes_extensions(item, arena, scratch) { + return false; + } + + let mut glob_idx = 0; + self.rest.iter().all(|c| { + evaluate( + item, + index, + c, + &self.glob, + &mut glob_idx, + false, + arena, + scratch, + ) + }) + } + + #[inline] + fn passes_extensions( + &self, + item: &T, + arena: ArenaPtr, + scratch: &mut ConstraintsBuffers, + ) -> bool { + if self.extensions.is_empty() { + return true; + } + item.write_file_name(arena, &mut scratch.fname); + self.extensions + .iter() + .any(|ext| file_has_extension(&scratch.fname, ext)) + } +} + +#[inline] +#[allow(clippy::too_many_arguments)] +fn evaluate( + item: &T, + index: usize, + constraint: &Constraint<'_>, + glob: &GlobStrategy, + glob_idx: &mut usize, + negate: bool, + arena: ArenaPtr, + scratch: &mut ConstraintsBuffers, +) -> bool { + let raw = match constraint { + Constraint::Glob(_) => { + let m = match glob { + GlobStrategy::None => true, + GlobStrategy::Prepass(masks) => masks + .get(*glob_idx) + .and_then(|mask| mask.get(index).copied()) + .unwrap_or(false), + GlobStrategy::Inline(patterns) => { + item.write_relative_path(arena, &mut scratch.path); + patterns + .get(*glob_idx) + .and_then(|p| p.as_ref()) + .map(|p| compiled_matches(p, &scratch.path)) + .unwrap_or(false) + } + }; + *glob_idx += 1; + m + } + // Reachable only via `Not(Extension(_))` — bare extensions are split out + // up front and handled in `passes_extensions`. + Constraint::Extension(ext) => { + item.write_file_name(arena, &mut scratch.fname); + file_has_extension(&scratch.fname, ext) + } + Constraint::PathSegment(segment) => { + item.write_relative_path(arena, &mut scratch.path); + path_contains_segment(&scratch.path, segment) + } + Constraint::FilePath(suffix) => { + item.write_relative_path(arena, &mut scratch.path); + path_ends_with_suffix(&scratch.path, suffix) + } + Constraint::Text(text) => { + // Only meaningful under negation (used as exclude filter). + item.write_relative_path(arena, &mut scratch.path); + find_case_insensitive_short(scratch.path.as_bytes(), text.as_bytes()).is_some() + } + Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter), + Constraint::Not(inner) => { + return evaluate(item, index, inner, glob, glob_idx, !negate, arena, scratch); + } + // Pass-throughs — handled at higher levels. + Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true, + }; + if negate { !raw } else { raw } +} + +#[inline] +fn matches_git_status(status: Option, filter: &GitStatusFilter) -> bool { + match (status, filter) { + (Some(s), GitStatusFilter::Modified) => is_modified_status(s), + (Some(s), GitStatusFilter::Untracked) => s.contains(git2::Status::WT_NEW), + (Some(s), GitStatusFilter::Staged) => s.intersects( + git2::Status::INDEX_NEW + | git2::Status::INDEX_MODIFIED + | git2::Status::INDEX_DELETED + | git2::Status::INDEX_RENAMED + | git2::Status::INDEX_TYPECHANGE, + ), + (Some(s), GitStatusFilter::Unmodified) => s.is_empty(), + (None, GitStatusFilter::Unmodified) => true, + (None, _) => false, + } +} + +#[inline] +#[cfg(feature = "zlob")] +fn compiled_matches(p: &GlobPattern, path: &str) -> bool { + p.matches_default(path) +} + +#[inline] +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +fn compiled_matches(p: &GlobPattern, path: &str) -> bool { + p.is_match(path) +} + +/// Decide between batch prepass and inline compiled patterns. +/// +/// `has_pre_filter` = true when something cheaper than glob can reject items first +/// (extensions OR non-glob constraints in `rest`). In that case inline pays glob +/// cost only on survivors and beats prepass on every workload we benched. Pure-glob +/// (no pre-filter) takes prepass — single batched zlob call beats N inline matches. +fn build_glob_strategy( + rest: &[&Constraint<'_>], + has_pre_filter: bool, + items: &[T], + arena: ArenaPtr, + overflow_arena: ArenaPtr, +) -> GlobStrategy { + if !contains_glob(rest) { + return GlobStrategy::None; + } + if has_pre_filter { + return GlobStrategy::Inline(compile_globs(rest)); + } + let buf = PathBuffer::collect(items, arena, overflow_arena); + let path_refs = buf.as_strs(); + GlobStrategy::Prepass(precompute_masks(rest, &path_refs)) +} + +/// `Glob` or `Not(Glob)` — the constraint kinds whose evaluation goes through +/// the GlobStrategy. Everything else can pre-reject items before glob runs. +fn is_glob_node(c: &Constraint<'_>) -> bool { + match c { + Constraint::Glob(_) => true, + Constraint::Not(inner) => is_glob_node(inner), + _ => false, + } +} + +fn contains_glob(rest: &[&Constraint<'_>]) -> bool { + rest.iter().any(|c| is_glob_node(c)) +} + +/// Contiguous byte buffer holding every item's `relative_path`. Single allocation +/// instead of N `String`s. On Windows the in-place pass folds `\\` -> `/` so the +/// glob library sees a canonical separator. +struct PathBuffer { + bytes: Vec, + offsets: Vec<(usize, usize)>, +} + +impl PathBuffer { + fn collect(items: &[T], arena: ArenaPtr, overflow_arena: ArenaPtr) -> Self { + let mut bytes = Vec::::new(); + let mut offsets = Vec::with_capacity(items.len()); + let mut tmp = String::with_capacity(64); + for item in items { + let item_arena = if item.is_overflow() { + overflow_arena + } else { + arena + }; + let start = bytes.len(); + item.write_relative_path(item_arena, &mut tmp); + bytes.extend_from_slice(tmp.as_bytes()); + offsets.push((start, bytes.len() - start)); + } + Self { bytes, offsets } + } + + fn as_strs(&self) -> Vec<&str> { + self.offsets + .iter() + .map(|&(off, len)| unsafe { + std::str::from_utf8_unchecked(&self.bytes[off..off + len]) + }) + .collect() + } +} + +fn precompute_masks(rest: &[&Constraint<'_>], paths: &[&str]) -> Vec> { + let mut out = Vec::new(); + for c in rest { + walk_globs(c, &mut |pattern| { + out.push(match_glob_pattern(pattern, paths)) + }); + } + out +} + +fn compile_globs(rest: &[&Constraint<'_>]) -> Vec> { + let mut out = Vec::new(); + for c in rest { + walk_globs(c, &mut |pattern| out.push(compile_one(pattern))); + } + out +} + +/// Visit every Glob (including ones nested under Not) in constraint walk order. +/// Order matters: `glob_idx` in the per-item evaluator increments by one per Glob node. +fn walk_globs(c: &Constraint<'_>, f: &mut F) { + match c { + Constraint::Glob(p) => f(p), + Constraint::Not(inner) => walk_globs(inner, f), + _ => {} + } +} + +#[cfg(feature = "zlob")] +fn compile_one(pattern: &str) -> Option { + zlob::ZlobPattern::compile(pattern, zlob::ZlobFlags::RECOMMENDED).ok() +} + +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +fn compile_one(pattern: &str) -> Option { + globset::Glob::new(pattern) + .ok() + .map(|g| g.compile_matcher()) +} + +/// Build a `paths.len()`-sized bitmap. Vec beats AHashSet ~2× in the per-item +/// filter loop — no hashing, plain array indexing, sequential prefetcher-friendly. +#[cfg(feature = "zlob")] +fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec { + let mut mask = vec![false; paths.len()]; + let Ok(hits) = zlob::zlob_match_paths_indices(pattern, paths, zlob::ZlobFlags::RECOMMENDED) + else { + return mask; + }; + for i in hits.to_iter() { + if i < mask.len() { + mask[i] = true; + } + } + mask +} + +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec { + let mut mask = vec![false; paths.len()]; + let Ok(glob) = globset::Glob::new(pattern) else { + return mask; + }; + let matcher = glob.compile_matcher(); + if paths.len() >= PAR_THRESHOLD { + use rayon::prelude::*; + mask.par_iter_mut() + .zip(paths.par_iter()) + .for_each(|(slot, p)| *slot = matcher.is_match(p)); + } else { + for (slot, p) in mask.iter_mut().zip(paths.iter()) { + *slot = matcher.is_match(p); + } + } + mask +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Clone)] + struct TestItem { + relative_path: &'static str, + file_name: &'static str, + } + + impl Constrainable for TestItem { + fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) { + out.clear(); + out.push_str(self.file_name); + } + + fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) { + out.clear(); + out.push_str(self.relative_path); + } + + fn git_status(&self) -> Option { + None + } + + fn is_overflow(&self) -> bool { + false + } + } + + #[test] + fn test_file_has_extension() { + assert!(file_has_extension("file.rs", "rs")); + assert!(file_has_extension("file.RS", "rs")); // case-insensitive + assert!(file_has_extension("file.test.rs", "rs")); + assert!(file_has_extension("a.rs", "rs")); + + assert!(!file_has_extension("file.tsx", "rs")); + assert!(!file_has_extension("rs", "rs")); // too short + assert!(!file_has_extension(".rs", "rs")); // just extension + assert!(!file_has_extension("file.rsx", "rs")); // different extension + assert!(!file_has_extension("filers", "rs")); // no dot + } + + #[test] + fn test_path_contains_segment() { + // Segment at start + assert!(path_contains_segment("src/lib.rs", "src")); + assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive + + // Segment in middle + assert!(path_contains_segment("app/src/lib.rs", "src")); + assert!(path_contains_segment("app/SRC/lib.rs", "src")); + + // Multiple levels + assert!(path_contains_segment("core/workflow/src/main.rs", "src")); + assert!(path_contains_segment( + "core/workflow/src/main.rs", + "workflow" + )); + assert!(path_contains_segment("core/workflow/src/main.rs", "core")); + + // Should not match partial segments + assert!(!path_contains_segment("source/lib.rs", "src")); + assert!(!path_contains_segment("mysrc/lib.rs", "src")); + + // Should not match filename + assert!(!path_contains_segment("lib/src", "src")); + + // Multi-segment constraints + assert!(path_contains_segment( + "libswscale/aarch64/input.S", + "libswscale/aarch64" + )); + assert!(path_contains_segment( + "foo/libswscale/aarch64/input.S", + "libswscale/aarch64" + )); + assert!(path_contains_segment( + "foo/LibSwscale/AArch64/input.S", + "libswscale/aarch64" + )); // case-insensitive + assert!(!path_contains_segment( + "xlibswscale/aarch64/input.S", + "libswscale/aarch64" + )); // partial match at start + assert!(!path_contains_segment( + "foo/libswscale/aarch64x/input.S", + "libswscale/aarch64" + )); // partial match at end + assert!(path_contains_segment( + "crates/fff-core/src/grep.rs", + "fff-core/src" + )); + + // Edge cases + assert!(!path_contains_segment("", "src")); + assert!(!path_contains_segment("src", "src")); // no trailing slash + } + + #[cfg(windows)] + #[test] + fn test_path_contains_segment_accepts_backslash() { + assert!(path_contains_segment("src\\lib.rs", "src")); + assert!(path_contains_segment( + "app\\modules\\src\\services\\x.lua", + "src" + )); + assert!(path_contains_segment("app\\SRC\\x.lua", "src")); + + assert!(path_contains_segment( + "foo\\libswscale\\aarch64\\input.S", + "libswscale/aarch64" + )); + assert!(path_contains_segment( + "crates\\fff-core\\src\\grep.rs", + "fff-core/src" + )); + + assert!(!path_contains_segment("mysrc\\lib.rs", "src")); + assert!(!path_contains_segment( + "xlibswscale\\aarch64\\in.S", + "libswscale/aarch64" + )); + } + + #[test] + fn test_path_ends_with_suffix() { + // Exact match + assert!(path_ends_with_suffix( + "libswscale/input.c", + "libswscale/input.c" + )); + + // Suffix match at / boundary + assert!(path_ends_with_suffix( + "foo/libswscale/input.c", + "libswscale/input.c" + )); + + // Deep nesting + assert!(path_ends_with_suffix( + "a/b/c/libswscale/input.c", + "libswscale/input.c" + )); + + // No boundary — partial directory name + assert!(!path_ends_with_suffix( + "xlibswscale/input.c", + "libswscale/input.c" + )); + + // Case insensitive + assert!(path_ends_with_suffix( + "foo/LibSwscale/Input.C", + "libswscale/input.c" + )); + + // Single file name + assert!(path_ends_with_suffix("input.c", "input.c")); + assert!(!path_ends_with_suffix("xinput.c", "input.c")); + + // Suffix longer than path + assert!(!path_ends_with_suffix("input.c", "foo/input.c")); + + // Simple path + assert!(path_ends_with_suffix("src/main.rs", "src/main.rs")); + assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs")); + } + + #[cfg(windows)] + #[test] + fn test_path_ends_with_suffix_accepts_backslash() { + assert!(path_ends_with_suffix( + "app\\modules\\src\\services\\handler.lua", + "services/handler.lua" + )); + assert!(path_ends_with_suffix( + "foo\\libswscale\\input.c", + "libswscale/input.c" + )); + assert!(!path_ends_with_suffix( + "xlibswscale\\input.c", + "libswscale/input.c" + )); + } + + #[test] + fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() { + assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c")); + assert!(path_ends_with_suffix( + "data/유니코드_파일_테스트.csv", + "유니코드_파일_테스트.csv" + )); + } + + #[test] + fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() { + assert!(!path_ends_with_suffix( + "dir/\u{2019}bar/file.txt", + "'bar/file.txt" + )); + } + + #[test] + fn test_path_ends_with_suffix_unicode_space_mismatch() { + assert!(!path_ends_with_suffix( + "dir/\u{202f}am/file.txt", + " am/file.txt" + )); + } + + #[test] + fn test_path_contains_segment_does_not_panic_on_unicode_segment() { + assert!(!path_contains_segment("문서/notes.txt", "문x")); + assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서")); + } + + #[test] + fn test_path_contains_segment_unicode_no_panic() { + assert!(!path_contains_segment( + "Library/Cloud/Project\u{2019}s Folder/books.ttl", + "Project's Folder" + )); + } + + #[test] + fn test_file_has_extension_unicode_no_panic() { + assert!(!file_has_extension("cat\u{00e9}.rs", "s")); + } + + #[test] + fn test_file_has_extension_unicode_filename() { + assert!(file_has_extension("운영-가이드.md", "md")); + assert!(file_has_extension("테스트.csv", "csv")); + assert!(!file_has_extension("테스트.csv", "md")); + } + + #[test] + fn test_apply_constraints_file_path_with_unicode_suffix() { + let arena_ptr = ArenaPtr(std::ptr::null()); + + let item = TestItem { + relative_path: "data/유니코드_파일_테스트.csv", + file_name: "유니코드_파일_테스트.csv", + }; + + let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")]; + let mismatch = [Constraint::FilePath("트.c")]; + + let exact_items = [item.clone()]; + let exact_matches = apply_constraints(&exact_items, &exact, arena_ptr, arena_ptr) + .expect("constraints applied"); + assert_eq!(exact_matches.len(), 1); + + let mismatch_items = [item]; + let mismatch_matches = apply_constraints(&mismatch_items, &mismatch, arena_ptr, arena_ptr) + .expect("constraints applied"); + assert!(mismatch_matches.is_empty()); + } + + #[test] + fn test_unicode_path_no_panic_real_korean_cases() { + // Real Korean paths that caused panics + let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"; + let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv"; + let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md"; + + // These must not panic regardless of segment/suffix used + assert!(!path_contains_segment(path1, "작성")); + assert!(!path_ends_with_suffix(path1, "작성.csv")); + assert!(!path_contains_segment(path2, "최종")); + assert!(!path_ends_with_suffix(path2, "최종.csv")); + assert!(!path_contains_segment(path3, "수정")); + assert!(!path_ends_with_suffix(path3, "것.md")); + + // Positive cases should still work + assert!(path_contains_segment( + path2, + "hermes-agent-lecture-materials" + )); + assert!(path_ends_with_suffix( + path1, + "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv" + )); + assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv")); + } + + #[test] + fn test_negated_glob_excludes_matching_files() { + let arena_ptr = ArenaPtr(std::ptr::null()); + + let items = vec![ + TestItem { + relative_path: "src/main.rs", + file_name: "main.rs", + }, + TestItem { + relative_path: "src/lib.ts", + file_name: "lib.ts", + }, + TestItem { + relative_path: "include/fff.h", + file_name: "fff.h", + }, + ]; + + // Not(Glob("**/*.rs")) should exclude .rs files + let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))]; + let result = apply_constraints(&items, &constraints, arena_ptr, arena_ptr).unwrap(); + let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect(); + assert!( + !paths.contains(&"src/main.rs"), + "rs file should be excluded" + ); + assert!(paths.contains(&"src/lib.ts"), "ts file should be included"); + assert!( + paths.contains(&"include/fff.h"), + "h file should be included" + ); + } + + #[test] + fn test_inline_glob_path_matches_prepass() { + // Mixed (extensions + glob) takes the inline-compiled path. + // Pure glob takes the prepass bitmap path. Both must give identical results. + let arena_ptr = ArenaPtr(std::ptr::null()); + let items = vec![ + TestItem { + relative_path: "src/main.rs", + file_name: "main.rs", + }, + TestItem { + relative_path: "src/lib.ts", + file_name: "lib.ts", + }, + TestItem { + relative_path: "tests/foo.rs", + file_name: "foo.rs", + }, + TestItem { + relative_path: "docs/readme.md", + file_name: "readme.md", + }, + ]; + + let mixed = vec![Constraint::Extension("rs"), Constraint::Glob("src/**")]; + let mixed_paths: Vec<&str> = apply_constraints(&items, &mixed, arena_ptr, arena_ptr) + .unwrap() + .iter() + .map(|i| i.relative_path) + .collect(); + assert_eq!(mixed_paths, vec!["src/main.rs"]); + + let pure_glob = vec![Constraint::Glob("src/**")]; + let glob_paths: Vec<&str> = apply_constraints(&items, &pure_glob, arena_ptr, arena_ptr) + .unwrap() + .iter() + .map(|i| i.relative_path) + .collect(); + assert!(glob_paths.contains(&"src/main.rs")); + assert!(glob_paths.contains(&"src/lib.ts")); + assert_eq!(glob_paths.len(), 2); + } + + #[test] + fn test_inline_negated_glob_with_extension() { + // Mixed Not(Glob) on inline path — exercise the negate=true branch in + // glob_matches_inline through the Not->Glob recursion. + let arena_ptr = ArenaPtr(std::ptr::null()); + let items = vec![ + TestItem { + relative_path: "src/main.rs", + file_name: "main.rs", + }, + TestItem { + relative_path: "vendor/foo.rs", + file_name: "foo.rs", + }, + TestItem { + relative_path: "vendor/foo.ts", + file_name: "foo.ts", + }, + ]; + + let constraints = vec![ + Constraint::Extension("rs"), + Constraint::Not(Box::new(Constraint::Glob("vendor/**"))), + ]; + let paths: Vec<&str> = apply_constraints(&items, &constraints, arena_ptr, arena_ptr) + .unwrap() + .iter() + .map(|i| i.relative_path) + .collect(); + assert_eq!(paths, vec!["src/main.rs"]); + } +} diff --git a/crates/fff-core/src/dbs/db_healthcheck.rs b/crates/fff-core/src/dbs/db_healthcheck.rs new file mode 100644 index 0000000..77d8349 --- /dev/null +++ b/crates/fff-core/src/dbs/db_healthcheck.rs @@ -0,0 +1,39 @@ +use crate::error::Result; + +/// Health information about a database +#[derive(Debug, Clone)] +pub struct DbHealth { + /// Path to the database file + pub path: String, + /// Size on disk in bytes + pub disk_size: u64, + /// Entry counts by table name + pub entry_counts: Vec<(&'static str, u64)>, + /// Set to `false` if can not acquire the write lock + pub healthy: bool, +} + +pub trait DbHealthChecker { + fn get_env(&self) -> &heed::Env; + fn is_healthy(&self) -> bool; + /// Entries per database, each group has a static string label + fn count_entries(&self) -> Result>; + + /// Health summary of the database, returns summary struct + fn get_health(&self) -> Result { + let env = self.get_env(); + + let size = env + .real_disk_size() + .map_err(crate::error::Error::GenericDbError)?; + let path = env.path().to_string_lossy().to_string(); + let entry_counts = self.count_entries()?; + + Ok(DbHealth { + path, + disk_size: size, + entry_counts, + healthy: self.is_healthy(), + }) + } +} diff --git a/crates/fff-core/src/dbs/frecency.rs b/crates/fff-core/src/dbs/frecency.rs new file mode 100644 index 0000000..7413396 --- /dev/null +++ b/crates/fff-core/src/dbs/frecency.rs @@ -0,0 +1,531 @@ +use super::db_healthcheck::DbHealthChecker; +use super::lmdb::{DbHealth, LmdbStore, is_map_full}; +use crate::error::{Error, Result}; +use crate::file_picker::FFFMode; +use crate::git::is_modified_status; +use heed::types::{Bytes, SerdeBincode}; +use heed::{Database, Env}; +use std::time::{SystemTime, UNIX_EPOCH}; +use std::{collections::VecDeque, path::Path}; + +const DECAY_CONSTANT: f64 = 0.0693; // ln(2)/10 for 10-day half-life +const SECONDS_PER_DAY: f64 = 86400.0; +const MAX_HISTORY_DAYS: f64 = 30.0; // Only consider accesses within 30 days +const MAX_TIMESTAMPS_PER_FILE: usize = 128; + +// AI mode: faster decay since AI sessions are shorter and more intense +const AI_DECAY_CONSTANT: f64 = 0.231; // ln(2)/3 for 3-day half-life +const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days + +#[derive(Debug)] +pub struct FrecencyTracker { + env: Env, + db: Database>>, + health: DbHealth, +} + +const MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [ + (16, 60 * 2), // 2 minutes + (8, 60 * 15), // 15 minutes + (4, 60 * 60), // 1 hour + (2, 60 * 60 * 24), // 1 day + (1, 60 * 60 * 24 * 7), // 1 week +]; + +// AI mode: compressed thresholds since AI edits happen in rapid bursts +const AI_MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [ + (16, 30), // 30 seconds + (8, 60 * 5), // 5 minutes + (4, 60 * 15), // 15 minutes + (2, 60 * 60), // 1 hour + (1, 60 * 60 * 4), // 4 hours +]; + +impl DbHealthChecker for FrecencyTracker { + fn get_env(&self) -> &heed::Env { + &self.env + } + + fn is_healthy(&self) -> bool { + self.health.is_healthy() + } + + fn count_entries(&self) -> Result> { + let rtxn = self + .env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + let count = self.db.len(&rtxn).map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + + Ok(vec![("absolute_frecency_entries", count)]) + } +} + +impl LmdbStore for FrecencyTracker { + const LABEL: &'static str = "frecency"; + // 10 MiB hard ceiling. Owner's db after years of use is ~560 KiB, so this + // leaves ~18× headroom while capping runaway growth (see GH issue #437). + const MAP_SIZE: usize = 10 * 1024 * 1024; + const MAX_DBS: u32 = 0; + // Nuke the db when it exceeds 8 MiB on disk — leaves a small margin under + // MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires. + const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024; + + fn env(&self) -> &Env { + &self.env + } + + fn health(&self) -> &DbHealth { + &self.health + } + + fn purge_stale_data(env: &Env) -> Result<()> { + let (deleted, pruned) = Self::purge_stale_entries(env)?; + if deleted > 0 || pruned > 0 { + tracing::info!(deleted, pruned, "Frecency GC purged entries"); + } + + Ok(()) + } +} + +impl FrecencyTracker { + /// Returns the on-disk path of the LMDB environment directory. + pub fn db_path(&self) -> &Path { + self.env.path() + } + + pub fn open(db_path: impl AsRef) -> Result { + let db_path = db_path.as_ref(); + let (env, health) = Self::open_env(db_path)?; + + let db = Self::open_database_safe(&env, None)?; + Ok(FrecencyTracker { db, env, health }) + } + + #[deprecated( + since = "0.7.0", + note = "LMDB unsafe no-lock mode is no longer supported; use `FrecencyTracker::open` instead. \ + The `_use_unsafe_no_lock` argument is ignored." + )] + pub fn new(db_path: impl AsRef, _use_unsafe_no_lock: bool) -> Result { + Self::open(db_path) + } + + /// Removes entries where all timestamps are older than MAX_HISTORY_DAYS, + /// and prunes stale timestamps from entries that still have recent ones. + /// Returns (deleted_count, pruned_count). + fn purge_stale_entries(env: &Env) -> Result<(usize, usize)> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64); + + let db: Database>> = Self::open_database_safe(env, None)?; + + let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + let mut to_delete: Vec> = Vec::new(); + let mut to_update: Vec<(Vec, VecDeque)> = Vec::new(); + + let iter = db.iter(&rtxn).map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + for result in iter { + let (key, accesses) = result.map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + + // Timestamps chronologically ordered (oldest at front). + let fresh_start = accesses.iter().position(|&ts| ts >= cutoff_time); + match fresh_start { + None => to_delete.push(key.to_vec()), + Some(0) => {} + Some(start) => { + let pruned: VecDeque = accesses.iter().skip(start).copied().collect(); + to_update.push((key.to_vec(), pruned)); + } + } + } + drop(rtxn); + + if to_delete.is_empty() && to_update.is_empty() { + return Ok((0, 0)); + } + + let mut wtxn = env.write_txn().map_err(|source| Error::DbStartWriteTxn { + db: Self::LABEL, + source, + })?; + + for key in &to_delete { + db.delete(&mut wtxn, key).map_err(|source| Error::DbWrite { + db: Self::LABEL, + source, + })?; + } + + for (key, accesses) in &to_update { + db.put(&mut wtxn, key, accesses) + .map_err(|source| Error::DbWrite { + db: Self::LABEL, + source, + })?; + } + wtxn.commit().map_err(|source| Error::DbCommit { + db: Self::LABEL, + source, + })?; + + Ok((to_delete.len(), to_update.len())) + } + + fn get_accesses(&self, path: &Path) -> Result>> { + let key_hash = Self::path_to_hash_bytes(path)?; + + let rtxn = self + .env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + let result = self + .db + .get(&rtxn, &key_hash) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + rtxn.commit().map_err(|source| Error::DbCommit { + db: Self::LABEL, + source, + })?; + + Ok(result) + } + + fn get_now(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + } + + fn path_to_hash_bytes(path: &Path) -> Result<[u8; 32]> { + // On Windows, resolve to the canonical form (short-name/case/symlink) + // so the same file always hashes to one key regardless of how the + // caller spelled it. Falls back to the raw path when the file no + // longer exists (e.g. watcher delete events), so the op is never + // dropped. No-op on other platforms. + #[cfg(windows)] + let canonical: Option = crate::path_utils::canonicalize(path).ok(); + #[cfg(windows)] + let path: &Path = canonical.as_deref().unwrap_or(path); + + let Some(key) = path.to_str() else { + return Err(Error::InvalidPath(path.to_path_buf())); + }; + + Ok(*blake3::hash(key.as_bytes()).as_bytes()) + } + + /// Returns seconds since the most recent tracked access, or `None` if the + /// file has never been tracked. + pub fn seconds_since_last_access(&self, path: &Path) -> Result> { + let accesses = self.get_accesses(path)?; + let last = accesses.and_then(|a| a.back().copied()); + Ok(last.map(|ts| self.get_now().saturating_sub(ts))) + } + + /// Number of tracked access for file path + pub fn access_count(&self, path: &Path) -> Result { + Ok(self.get_accesses(path)?.map_or(0, |a| a.len())) + } + + pub fn track_access(&self, path: &Path) -> Result<()> { + let key_hash = Self::path_to_hash_bytes(path)?; + let mut accesses = self.get_accesses(path)?.unwrap_or_default(); + + let now = self.get_now(); + let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64); + + // Drop stale timestamps from the front while also enforcing the + // per-file cap. Reserves one slot for the `push_back` below. + while let Some(&front_time) = accesses.front() { + if front_time < cutoff_time || accesses.len() >= MAX_TIMESTAMPS_PER_FILE { + accesses.pop_front(); + } else { + break; + } + } + + accesses.push_back(now); + tracing::debug!(?path, accesses = accesses.len(), "Tracking access"); + + let mut wtxn = self + .env + .write_txn() + .map_err(|source| Error::DbStartWriteTxn { + db: Self::LABEL, + source, + })?; + if let Err(e) = self.db.put(&mut wtxn, &key_hash, &accesses) { + if is_map_full(&e) { + self.health.mark_unhealthy("MDB_MAP_FULL on put"); + tracing::error!( + ?path, + "Frecency DB hit MDB_MAP_FULL; dropping write — db will be \ + erased on next open via LmdbStore::erase_if_oversized" + ); + return Ok(()); + } + return Err(Error::DbWrite { + db: Self::LABEL, + source: e, + }); + } + + wtxn.commit() + .inspect_err(|e| { + if is_map_full(e) { + self.health.mark_unhealthy("MDB_MAP_FULL on commit"); + tracing::error!( + ?path, + "Frecency DB hit MDB_MAP_FULL on commit; dropping write" + ); + } + }) + .map_err(|source| Error::DbCommit { + db: Self::LABEL, + source, + }) + } + + pub fn get_access_score(&self, file_path: &Path, mode: FFFMode) -> i64 { + let accesses = self + .get_accesses(file_path) + .ok() + .flatten() + .unwrap_or_default(); + + if accesses.is_empty() { + return 0; + } + + let decay_constant = if mode.is_ai() { + AI_DECAY_CONSTANT + } else { + DECAY_CONSTANT + }; + let max_history_days = if mode.is_ai() { + AI_MAX_HISTORY_DAYS + } else { + MAX_HISTORY_DAYS + }; + + let now = self.get_now(); + let mut total_frecency = 0.0; + + let cutoff_time = now.saturating_sub((max_history_days * SECONDS_PER_DAY) as u64); + + for &access_time in accesses.iter().rev() { + if access_time < cutoff_time { + break; // All remaining entries are older, stop processing + } + + let days_ago = (now.saturating_sub(access_time) as f64) / SECONDS_PER_DAY; + let decay_factor = (-decay_constant * days_ago).exp(); + total_frecency += decay_factor; + } + + let normalized_frecency = if total_frecency <= 10.0 { + total_frecency + } else { + 10.0 + (total_frecency - 10.0).sqrt() // Diminishing: >10 accesses grow slowly + }; + + normalized_frecency.round() as i64 + } + + /// Calculating modification score but only if the file is modified in the current git dir + pub fn get_modification_score( + &self, + modified_time: u64, + git_status: Option, + mode: FFFMode, + ) -> i64 { + let is_modified_git_status = git_status.is_some_and(is_modified_status); + if !is_modified_git_status { + return 0; + } + + let thresholds = if mode.is_ai() { + &AI_MODIFICATION_THRESHOLDS + } else { + &MODIFICATION_THRESHOLDS + }; + + let now = self.get_now(); + let duration_since = now.saturating_sub(modified_time); + + for i in 0..thresholds.len() { + let (current_points, current_threshold) = thresholds[i]; + + if duration_since <= current_threshold { + if i == 0 || duration_since == current_threshold { + return current_points; + } + + let (prev_points, prev_threshold) = thresholds[i - 1]; + + let time_range = current_threshold - prev_threshold; + let time_offset = duration_since - prev_threshold; + let points_diff = prev_points - current_points; + + let interpolated_score = + prev_points - (points_diff * time_offset as i64) / time_range as i64; + + return interpolated_score; + } + } + + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_picker::FFFMode; + + // A path that doesn't exist on disk must still hash (canonicalize fails on + // Windows → falls back to the raw string), so watcher delete events and + // raced files never drop their frecency op. + #[test] + fn hashes_nonexistent_path_without_error() { + let missing = Path::new("/this/path/definitely/does/not/exist/frecency_test_xyz"); + assert!(FrecencyTracker::path_to_hash_bytes(missing).is_ok()); + } + + fn calculate_test_frecency_score(access_timestamps: &[u64], current_time: u64) -> i64 { + let mut total_frecency = 0.0; + + for &access_time in access_timestamps { + let days_ago = (current_time.saturating_sub(access_time) as f64) / SECONDS_PER_DAY; + let decay_factor = (-DECAY_CONSTANT * days_ago).exp(); + total_frecency += decay_factor; + } + + let normalized_frecency = if total_frecency <= 20.0 { + total_frecency + } else { + 20.0 + (total_frecency - 10.0).sqrt() + }; + + normalized_frecency.round() as i64 + } + + #[test] + fn test_frecency_calculation() { + let current_time = 1000000000; // Base timestamp + + let score = calculate_test_frecency_score(&[], current_time); + assert_eq!(score, 0); + + let accesses = [current_time]; // Accessed right now + let score = calculate_test_frecency_score(&accesses, current_time); + assert_eq!(score, 1); // 1.0 decay factor = 1 + + let ten_days_seconds = 10 * 86400; // 10 days in seconds + let accesses = [current_time - ten_days_seconds]; + let score = calculate_test_frecency_score(&accesses, current_time); + assert_eq!(score, 1); // ~0.5 decay factor rounds to 1 + + let accesses = [ + current_time, // Today + current_time - 86400, // 1 day ago + current_time - 172800, // 2 days ago + ]; + let score = calculate_test_frecency_score(&accesses, current_time); + assert!(score > 2 && score < 4, "Score: {}", score); // About 3 accesses with decay + + let thirty_days = 30 * 86400; + let accesses = [current_time - thirty_days]; // 30 days ago + let score = calculate_test_frecency_score(&accesses, current_time); + assert!( + score < 2, + "Old access should have minimal score, got: {}", + score + ); + + let recent_frequent = [current_time, current_time - 86400, current_time - 172800]; + let old_single = [current_time - ten_days_seconds]; + + let recent_score = calculate_test_frecency_score(&recent_frequent, current_time); + let old_score = calculate_test_frecency_score(&old_single, current_time); + + assert!( + recent_score > old_score, + "Recent frequent access ({}) should score higher than old single access ({})", + recent_score, + old_score + ); + } + + #[test] + fn test_modification_score_interpolation() { + let temp_dir = std::env::temp_dir().join("fff_test_interpolation"); + let _ = std::fs::remove_dir_all(&temp_dir); + let tracker = FrecencyTracker::open(temp_dir.to_str().unwrap()).unwrap(); + + let current_time = tracker.get_now(); + let git_status = Some(git2::Status::WT_MODIFIED); + + // At 5 minutes: should interpolate between 16 and 8 points + let five_minutes_ago = current_time - (5 * 60); + let score = tracker.get_modification_score(five_minutes_ago, git_status, FFFMode::Neovim); + + // Expected: 16 - (8 * 3 / 13) = 16 - 1 = 15 points + // (time_offset = 5-2 = 3, time_range = 15-2 = 13, points_diff = 16-8 = 8) + assert_eq!(score, 15, "5 minutes should interpolate to 15 points"); + + let two_minutes_ago = current_time - (2 * 60); + let score = tracker.get_modification_score(two_minutes_ago, git_status, FFFMode::Neovim); + assert_eq!(score, 16, "2 minutes should be exactly 16 points"); + + let fifteen_minutes_ago = current_time - (15 * 60); + let score = + tracker.get_modification_score(fifteen_minutes_ago, git_status, FFFMode::Neovim); + assert_eq!(score, 8, "15 minutes should be exactly 8 points"); + + // At 12 hours: should interpolate between 4 and 2 points + let twelve_hours_ago = current_time - (12 * 60 * 60); + let score = tracker.get_modification_score(twelve_hours_ago, git_status, FFFMode::Neovim); + // Expected: 4 - (2 * 11 / 23) = 4 - 0 = 4 points (integer division) + // (time_offset = 12-1 = 11 hours, time_range = 24-1 = 23 hours, points_diff = 4-2 = 2) + assert_eq!(score, 4, "12 hours should interpolate to 4 points"); + + // at 18 hours for more significant interpolation + let eighteen_hours_ago = current_time - (18 * 60 * 60); + let score = tracker.get_modification_score(eighteen_hours_ago, git_status, FFFMode::Neovim); + // Expected: 4 - (2 * 17 / 23) = 4 - 1 = 3 points + assert_eq!(score, 3, "18 hours should interpolate to 3 points"); + + let score = tracker.get_modification_score(five_minutes_ago, None, FFFMode::Neovim); + assert_eq!(score, 0, "No git status should return 0"); + + let _ = std::fs::remove_dir_all(&temp_dir); + } +} diff --git a/crates/fff-core/src/dbs/lmdb.rs b/crates/fff-core/src/dbs/lmdb.rs new file mode 100644 index 0000000..9eba707 --- /dev/null +++ b/crates/fff-core/src/dbs/lmdb.rs @@ -0,0 +1,259 @@ +use heed::{Database, Env, EnvOpenOptions}; +use std::fs; +use std::path::Path; +use std::sync::Arc; +use std::sync::RwLock; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::thread; +use std::time::Duration; + +use crate::error::{Error, Result}; + +pub(crate) fn is_map_full(err: &heed::Error) -> bool { + matches!(err, heed::Error::Mdb(heed::MdbError::MapFull)) +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DbHealthState { + Pending = 0, + Healthy = 1, + Degraded = 2, +} + +impl DbHealthState { + fn from_u8(v: u8) -> Self { + debug_assert!(v <= 2); + + match v { + 0 => Self::Pending, + 1 => Self::Healthy, + _ => Self::Degraded, + } + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct DbHealth(Arc); + +impl DbHealth { + pub(crate) fn new() -> Self { + Self(Arc::new(AtomicU8::new(DbHealthState::Pending as u8))) + } + + pub(crate) fn is_healthy(&self) -> bool { + // Pending counts as unhealthy: if the GC thread never flipped to + // Healthy, something's wrong (deadlocked clear_stale_readers, stuck + // writer mutex, etc.) and we want that surfaced to the user. + DbHealthState::from_u8(self.0.load(Ordering::Acquire)) == DbHealthState::Healthy + } + + pub(crate) fn mark_healthy(&self) { + let _ = self.0.compare_exchange( + DbHealthState::Pending as u8, + DbHealthState::Healthy as u8, + Ordering::AcqRel, + Ordering::Acquire, + ); + } + + pub(crate) fn mark_unhealthy(&self, reason: &'static str) { + let prev = self.0.swap(DbHealthState::Degraded as u8, Ordering::AcqRel); + if DbHealthState::from_u8(prev) != DbHealthState::Degraded { + tracing::error!(reason, "LMDB tracker marked unhealthy"); + } + } +} + +/// Spawns a background thread that is ensuring that the environment that was previously +/// open is safe, accessible and doesn't have a corrupted lock.md file. If it does this thread will +/// hang indefinitely but we will have the information that the database is in failure mode +pub(crate) fn spawn_lmdb_gc(shared: Arc>>) { + let thread_shared = shared.clone(); + let spawn_result = thread::Builder::new() + .name("fff-lmdb-gc".into()) + .spawn(move || { + // Holding a read guard blocks `destroy` / re-init's write + // guard until this thread finishes — natural serialization. + let guard = match thread_shared.read() { + Ok(g) => g, + Err(e) => { + tracing::debug!("gc: read lock poisoned: {e}"); + return; + } + }; + let Some(ref tracker) = *guard else { + return; // destroyed before we started + }; + let env = tracker.env(); + + if let Err(e) = T::purge_stale_data(env) { + tracing::debug!("purge_stale_data failed: {e}"); + } + + tracker.health().mark_healthy(); + }); + + if let Err(e) = spawn_result { + tracing::debug!(?e, "failed to spawn fff-lmdb-gc thread"); + // No thread = mark healthy now so healthcheck isn't stuck Pending. + if let Ok(guard) = shared.read() + && let Some(ref tracker) = *guard + { + tracker.health().mark_healthy(); + } + } +} + +// Concurrent `mdb_env_open` calls on the same path can race on macOS +// this is for some reason fixabtly by simple retry of the open +fn is_transient_env_open_error(err: &heed::Error) -> bool { + match err { + heed::Error::Io(io) => matches!( + io.kind(), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound + ), + _ => false, + } +} + +pub(crate) trait LmdbStore: Sized + Send + Sync + 'static { + /// Short label used to defferintiate different instances of this trait + const LABEL: &'static str; + /// LMDB map size in bytes. Must be a multiple of the OS page size. + const MAP_SIZE: usize; + /// Number of named sub-databases. `0` for single-db envs. + const MAX_DBS: u32; + /// Hard cap on `data.mdb` size. + const SIZE_CAP_BYTES: u64; + + /// Borrow the env in the read lock + fn env(&self) -> &Env; + /// Borrow the health flag from the tracker. + fn health(&self) -> &DbHealth; + + /// Override to purge stale rows, compact, etc. Default no-op. Runs on + /// the GC thread while a read lock is held against the shared handle, + /// so destroy / re-init naturally wait for it. + fn purge_stale_data(_env: &Env) -> Result<()> { + Ok(()) + } + + /// Open the LMDB env. Returns env + a `DbHealth` starting in Pending; + /// the GC thread spawned by `spawn_gc` flips it to Healthy. Write + /// paths flip it to Degraded on MDB_MAP_FULL. + #[tracing::instrument] + fn open_env(db_path: &Path) -> Result<(Env, DbHealth)> { + Self::erase_if_oversized(db_path); + fs::create_dir_all(db_path).map_err(Error::CreateDir)?; + let db = Self::LABEL; + + const MAX_ATTEMPTS: u32 = 8; + let mut attempt = 0u32; + let env = loop { + let result = unsafe { + let mut opts = EnvOpenOptions::new(); + opts.map_size(Self::MAP_SIZE); + if Self::MAX_DBS > 0 { + opts.max_dbs(Self::MAX_DBS); + } + opts.open(db_path) + }; + + match result { + Ok(env) => break env, + Err(e) if is_transient_env_open_error(&e) && attempt + 1 < MAX_ATTEMPTS => { + attempt += 1; + tracing::debug!( + path = %db_path.display(), + attempt, + error = ?e, + "transient LMDB env open error, retrying" + ); + + thread::sleep(Duration::from_millis(50)); + } + Err(e) => return Err(Error::EnvOpen { db, source: e }), + } + }; + + // Reclaim reader slots left behind by prior processes that died + // without cleanup. Must run before we start any read txns (which + // open_database_safe does) — otherwise we may hit MDB_READERS_FULL + // on a fresh env just because lock.mdb still has stale entries + // from a previous crash. + // + // This is the one LMDB maintenance call we run on the caller's + // thread. If the lock file is genuinely wedged this will block + // forever, but the alternative — never getting past init — is + // worse and the bg-thread trick doesn't solve it anyway. + match env.clear_stale_readers() { + Ok(cleared) if cleared > 0 => { + tracing::warn!(cleared, "reclaimed stale LMDB reader slots at open"); + } + Ok(_) => {} + Err(e) => tracing::debug!("clear_stale_readers at open failed: {e}"), + } + + Ok((env, DbHealth::new())) + } + + /// Open or create a database without blocking on the LMDB writer mutex + /// when the database already exists. + fn open_database_safe(env: &Env, name: Option<&str>) -> Result> + where + KC: 'static, + DC: 'static, + { + let db = Self::LABEL; + let rtxn = env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { db, source })?; + let maybe_db: Option> = env + .open_database(&rtxn, name) + .map_err(|source| Error::DbOpen { db, source })?; + + // do not drop the DB here + rtxn.commit() + .map_err(|source| Error::DbCommit { db, source })?; + + match maybe_db { + Some(handle) => Ok(handle), + None => { + // First time: create the database (requires write lock). + // unfortunately this CAN be deadlocking and this is what we see happens + // if the other part of the code is segfaulting, so the only rule to prevent this + // write the good code mf, okay? + let mut wtxn = env + .write_txn() + .map_err(|source| Error::DbStartWriteTxn { db, source })?; + let handle = env + .create_database(&mut wtxn, name) + .map_err(|source| Error::DbCreate { db, source })?; + + wtxn.commit() + .map_err(|source| Error::DbCommit { db, source })?; + Ok(handle) + } + } + } + + fn erase_if_oversized(db_path: &Path) { + let data = db_path.join("data.mdb"); + let Ok(meta) = fs::metadata(&data) else { + return; + }; + if meta.len() <= Self::SIZE_CAP_BYTES { + return; + } + + tracing::error!( + path = %db_path.display(), + size = meta.len(), + cap = Self::SIZE_CAP_BYTES, + "LMDB db exceeds size cap, erasing" + ); + let _ = fs::remove_file(&data); + let _ = fs::remove_file(db_path.join("lock.mdb")); + } +} diff --git a/crates/fff-core/src/dbs/mod.rs b/crates/fff-core/src/dbs/mod.rs new file mode 100644 index 0000000..63073cc --- /dev/null +++ b/crates/fff-core/src/dbs/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod lmdb; + +pub mod db_healthcheck; +pub use db_healthcheck::{DbHealth, DbHealthChecker}; + +pub mod frecency; +pub use frecency::*; + +pub mod query_tracker; +pub use query_tracker::*; diff --git a/crates/fff-core/src/dbs/query_tracker.rs b/crates/fff-core/src/dbs/query_tracker.rs new file mode 100644 index 0000000..40d0117 --- /dev/null +++ b/crates/fff-core/src/dbs/query_tracker.rs @@ -0,0 +1,548 @@ +use super::db_healthcheck::DbHealthChecker; +use super::lmdb::{DbHealth, LmdbStore, is_map_full}; +use crate::error::Error; +use heed::types::{Bytes, SerdeBincode}; +use heed::{Database, Env}; +use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const MAX_HISTORY_ENTRIES: usize = 128; + +/// Simplified QueryFileEntry without redundant fields +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct QueryMatchEntry { + pub file_path: PathBuf, // File that was actually opened + pub open_count: u32, // Number of times opened with this query + pub last_opened: u64, // Unix timestamp +} + +/// Entry for query history tracking +#[derive(Debug, Serialize, Deserialize, Clone)] +struct HistoryEntry { + query: String, + timestamp: u64, +} + +#[derive(Debug)] +pub struct QueryTracker { + env: Env, + // Database for (project_path, query) -> QueryMatchEntry mappings + query_file_db: Database>, + // Database for project_path -> VecDeque mappings (file picker) + query_history_db: Database>>, + // Database for project_path -> VecDeque mappings (grep) + grep_query_history_db: Database>>, + health: DbHealth, +} + +impl DbHealthChecker for QueryTracker { + fn get_env(&self) -> &Env { + &self.env + } + + fn is_healthy(&self) -> bool { + self.health.is_healthy() + } + + fn count_entries(&self) -> Result, Error> { + let rtxn = self + .env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + + let count_queries = self + .query_file_db + .len(&rtxn) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + let count_histories = self + .query_history_db + .len(&rtxn) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + let count_grep_histories = + self.grep_query_history_db + .len(&rtxn) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + + Ok(vec![ + ("query_file_entries", count_queries), + ("query_history_entries", count_histories), + ("grep_query_history_entries", count_grep_histories), + ]) + } +} + +impl LmdbStore for QueryTracker { + const LABEL: &'static str = "query"; + // 10 MiB hard ceiling. Same reasoning as FrecencyTracker (GH issue #437). + const MAP_SIZE: usize = 10 * 1024 * 1024; + const MAX_DBS: u32 = 16; + const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024; + + fn env(&self) -> &Env { + &self.env + } + + fn health(&self) -> &DbHealth { + &self.health + } +} + +impl QueryTracker { + /// Returns the on-disk path of the LMDB environment directory. + pub fn db_path(&self) -> &Path { + self.env.path() + } + + pub fn open(db_path: impl AsRef) -> Result { + let db_path = db_path.as_ref(); + let (env, health) = Self::open_env(db_path)?; + + let query_file_db = Self::open_database_safe(&env, Some("query_file_associations"))?; + let query_history_db = Self::open_database_safe(&env, Some("query_history"))?; + let grep_query_history_db = Self::open_database_safe(&env, Some("grep_query_history"))?; + + Ok(QueryTracker { + env, + query_file_db, + query_history_db, + grep_query_history_db, + health, + }) + } + + #[deprecated( + since = "0.7.0", + note = "LMDB unsafe no-lock mode is no longer supported; use `QueryTracker::open` instead. \ + The `_use_unsafe_no_lock` argument is ignored." + )] + pub fn new(db_path: impl AsRef, _use_unsafe_no_lock: bool) -> Result { + Self::open(db_path) + } + + fn get_now(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + } + + fn create_query_key(project_path: &Path, query: &str) -> Result<[u8; 32], Error> { + let project_str = project_path + .to_str() + .ok_or_else(|| Error::InvalidPath(project_path.to_path_buf()))?; + + let mut hasher = blake3::Hasher::default(); + hasher.update(project_str.as_bytes()); + hasher.update(b"::"); + hasher.update(query.as_bytes()); + + Ok(*hasher.finalize().as_bytes()) + } + + fn create_project_key(project_path: &Path) -> Result<[u8; 32], Error> { + let project_str = project_path + .to_str() + .ok_or_else(|| Error::InvalidPath(project_path.to_path_buf()))?; + + Ok(*blake3::hash(project_str.as_bytes()).as_bytes()) + } + + /// Append a query to a history database within an existing write transaction. + fn append_to_history( + db: &Database>>, + wtxn: &mut heed::RwTxn, + project_key: &[u8; 32], + query: &str, + now: u64, + ) -> Result<(), Error> { + let mut history = db + .get(wtxn, project_key) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })? + .unwrap_or_default(); + + history.push_back(HistoryEntry { + query: query.to_string(), + timestamp: now, + }); + while history.len() > MAX_HISTORY_ENTRIES { + history.pop_front(); + } + + db.put(wtxn, project_key, &history) + .map_err(|source| Error::DbWrite { + db: Self::LABEL, + source, + })?; + Ok(()) + } + + /// Read a query from a history database at a specific offset. + /// offset=0 returns most recent, offset=1 returns 2nd most recent, etc. + fn read_history_at_offset( + db: &Database>>, + env: &Env, + project_key: &[u8; 32], + offset: usize, + ) -> Result, Error> { + let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + + let mut history = db + .get(&rtxn, project_key) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })? + .unwrap_or_default(); + + // history is FIFO, last element is most recent + if history.len() > offset { + let index = history.len() - 1 - offset; + let record = history.remove(index); + Ok(record.map(|r| r.query)) + } else { + Ok(None) + } + } + + pub fn track_query_completion( + &mut self, + query: &str, + project_path: &Path, + file_path: &Path, + ) -> Result<(), Error> { + let now = self.get_now(); + let file_path_buf = file_path.to_path_buf(); + + let query_key = Self::create_query_key(project_path, query)?; + let mut wtxn = self + .env + .write_txn() + .map_err(|source| Error::DbStartWriteTxn { + db: Self::LABEL, + source, + })?; + + let mut entry = self + .query_file_db + .get(&wtxn, &query_key) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })? + .unwrap_or_else(|| QueryMatchEntry { + file_path: file_path_buf.clone(), + open_count: 0, + last_opened: now, + }); + + if entry.file_path == file_path_buf { + tracing::debug!( + ?query, + ?file_path, + "Query completed for same file as last time" + ); + + // Same file - just increment count + entry.open_count += 1; + } else { + tracing::debug!( + ?query, + ?file_path, + "Query completed for different file than last time" + ); + + // Different file - replace and reset count to 1 + entry.file_path = file_path_buf; + entry.open_count = 1; + } + + entry.last_opened = now; + + if let Err(e) = self.query_file_db.put(&mut wtxn, &query_key, &entry) { + if is_map_full(&e) { + self.health.mark_unhealthy("MDB_MAP_FULL on put"); + tracing::error!( + ?query, + "Query tracker DB hit MDB_MAP_FULL; dropping write — db will \ + be erased on next open" + ); + return Ok(()); + } + return Err(Error::DbWrite { + db: Self::LABEL, + source: e, + }); + } + + // Update query history database + let project_key = Self::create_project_key(project_path)?; + if let Err(e) = + Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now) + { + if let Error::DbWrite { + source: ref inner, .. + } = e + && is_map_full(inner) + { + self.health.mark_unhealthy("MDB_MAP_FULL on history append"); + tracing::error!(?query, "Query tracker DB map full while appending history"); + return Ok(()); + } + return Err(e); + } + + if let Err(e) = wtxn.commit() { + if is_map_full(&e) { + self.health.mark_unhealthy("MDB_MAP_FULL on commit"); + tracing::error!(?query, "Query tracker DB map full on commit"); + return Ok(()); + } + return Err(Error::DbCommit { + db: Self::LABEL, + source: e, + }); + } + + tracing::debug!(?query, ?file_path, "Tracked query completion"); + Ok(()) + } + + pub fn get_last_query_entry( + &self, + query: &str, + project_path: &Path, + min_combo_count: u32, + ) -> Result, Error> { + let query_key = Self::create_query_key(project_path, query)?; + let rtxn = self + .env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + + let last_match = self + .query_file_db + .get(&rtxn, &query_key) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })?; + + Ok(last_match.filter(|entry| entry.open_count >= min_combo_count)) + } + + pub fn get_last_query_path( + &self, + query: &str, + project_path: &Path, + file_path: &Path, + combo_boost: i32, + ) -> Result { + let query_key = Self::create_query_key(project_path, query)?; + tracing::debug!(?query_key, "HASH"); + let rtxn = self + .env + .read_txn() + .map_err(|source| Error::DbStartReadTxn { + db: Self::LABEL, + source, + })?; + + match self + .query_file_db + .get(&rtxn, &query_key) + .map_err(|source| Error::DbRead { + db: Self::LABEL, + source, + })? { + Some(entry) => { + // Check if the file path matches and return boost + if entry.file_path == file_path && entry.open_count >= 2 { + Ok(combo_boost) + } else { + Ok(0) + } + } + None => Ok(0), // Query not found + } + } + + /// Get query from file picker history at a specific offset. + /// offset=0 returns most recent query, offset=1 returns 2nd most recent, etc. + pub fn get_historical_query( + &self, + project_path: &Path, + offset: usize, + ) -> Result, Error> { + let project_key = Self::create_project_key(project_path)?; + Self::read_history_at_offset(&self.query_history_db, &self.env, &project_key, offset) + } + + /// Track a grep query in the grep-specific history. + /// Only records query history (no file association tracking needed for grep). + pub fn track_grep_query(&mut self, query: &str, project_path: &Path) -> Result<(), Error> { + let now = self.get_now(); + let project_key = Self::create_project_key(project_path)?; + let mut wtxn = self + .env + .write_txn() + .map_err(|source| Error::DbStartWriteTxn { + db: Self::LABEL, + source, + })?; + + if let Err(e) = Self::append_to_history( + &self.grep_query_history_db, + &mut wtxn, + &project_key, + query, + now, + ) { + if let Error::DbWrite { + source: ref inner, .. + } = e + && is_map_full(inner) + { + self.health + .mark_unhealthy("MDB_MAP_FULL on grep history append"); + tracing::error!(?query, "Grep query history DB map full; dropping write"); + return Ok(()); + } + return Err(e); + } + + if let Err(e) = wtxn.commit() { + if is_map_full(&e) { + self.health.mark_unhealthy("MDB_MAP_FULL on commit"); + tracing::error!(?query, "Grep query history DB map full on commit"); + return Ok(()); + } + return Err(Error::DbCommit { + db: Self::LABEL, + source: e, + }); + } + + tracing::debug!(?query, "Tracked grep query"); + Ok(()) + } + + /// Get grep query from history at a specific offset. + /// offset=0 returns most recent grep query, offset=1 returns 2nd most recent, etc. + pub fn get_historical_grep_query( + &self, + project_path: &Path, + offset: usize, + ) -> Result, Error> { + let project_key = Self::create_project_key(project_path)?; + Self::read_history_at_offset(&self.grep_query_history_db, &self.env, &project_key, offset) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + #[test] + fn test_query_tracking() { + let temp_dir = env::temp_dir().join("fff_test_query_tracking_new"); + let _ = std::fs::remove_dir_all(&temp_dir); + + let mut tracker = QueryTracker::open(temp_dir.to_str().unwrap()).unwrap(); + + let project_path = PathBuf::from("/test/project"); + let file_path = PathBuf::from("/test/project/src/main.rs"); + + // First completion + tracker + .track_query_completion("main", &project_path, &file_path) + .unwrap(); + let boost = tracker + .get_last_query_path("main", &project_path, &file_path, 10000) + .unwrap(); + assert_eq!(boost, 0, "First completion should not boost"); + + // Second completion - should boost now + tracker + .track_query_completion("main", &project_path, &file_path) + .unwrap(); + let boost = tracker + .get_last_query_path("main", &project_path, &file_path, 10000) + .unwrap(); + assert_eq!(boost, 10000, "Second completion should boost"); + + // Different file for same query - should reset count and no boost + let other_file = PathBuf::from("/test/project/src/lib.rs"); + tracker + .track_query_completion("main", &project_path, &other_file) + .unwrap(); + let boost = tracker + .get_last_query_path("main", &project_path, &other_file, 10000) + .unwrap(); + assert_eq!(boost, 0, "Different file should reset boost"); + + // Original file should no longer get boost (replaced by new file) + let boost = tracker + .get_last_query_path("main", &project_path, &file_path, 10000) + .unwrap(); + assert_eq!(boost, 0, "Original file should not boost after replacement"); + + let _ = std::fs::remove_dir_all(&temp_dir); + } + + #[test] + fn test_hashing_functions() { + let project_path = PathBuf::from("/test/project"); + + // Test project key hashing + let key1 = QueryTracker::create_project_key(&project_path).unwrap(); + let key2 = QueryTracker::create_project_key(&project_path).unwrap(); + assert_eq!(key1, key2, "Same project should hash to same key"); + + // Test query key hashing + let query_key1 = QueryTracker::create_query_key(&project_path, "test").unwrap(); + let query_key2 = QueryTracker::create_query_key(&project_path, "test").unwrap(); + assert_eq!( + query_key1, query_key2, + "Same project+query should hash to same key" + ); + + // Different queries should hash differently + let query_key3 = QueryTracker::create_query_key(&project_path, "different").unwrap(); + assert_ne!( + query_key1, query_key3, + "Different queries should hash to different keys" + ); + + // Different projects should hash differently + let other_project = PathBuf::from("/other/project"); + let query_key4 = QueryTracker::create_query_key(&other_project, "test").unwrap(); + assert_ne!( + query_key1, query_key4, + "Different projects should hash to different keys" + ); + } +} diff --git a/crates/fff-core/src/error.rs b/crates/fff-core/src/error.rs new file mode 100644 index 0000000..451af79 --- /dev/null +++ b/crates/fff-core/src/error.rs @@ -0,0 +1,99 @@ +use std::path::StripPrefixError; + +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum Error { + #[error("Thread panicked")] + ThreadPanic, + #[error("Invalid path {0}")] + InvalidPath(std::path::PathBuf), + #[error( + "Can not run certain FFF features in a file system root or home directories. Consider smaller per-project directories." + )] + FilesystemRoot(std::path::PathBuf), + #[error("File picker not initialized")] + FilePickerMissing, + #[error("Failed to acquire lock for frecency")] + AcquireFrecencyLock, + #[error("Failed to acquire lock for items by provider")] + AcquireItemLock, + #[error("Failed to acquire lock for path cache")] + AcquirePathCacheLock, + #[error("Failed to create directory: {0}")] + CreateDir(#[from] std::io::Error), + #[error("Failed to remove database directory {path}: {source}")] + RemoveDbDir { + path: std::path::PathBuf, + source: std::io::Error, + }, + #[error("Something is wrong with the local db instance: {0}")] + GenericDbError(#[from] heed::Error), + #[error("Failed to open {db} database env: {source}")] + EnvOpen { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to create {db} database: {source}")] + DbCreate { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to open {db} database: {source}")] + DbOpen { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to clear stale readers for {db} database: {source}")] + DbClearStaleReaders { + db: &'static str, + #[source] + source: heed::Error, + }, + + #[error("Failed to start read transaction for {db} database: {source}")] + DbStartReadTxn { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to start write transaction for {db} database: {source}")] + DbStartWriteTxn { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to read from {db} database: {source}")] + DbRead { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to write to {db} database: {source}")] + DbWrite { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to commit write transaction to {db} database: {source}")] + DbCommit { + db: &'static str, + #[source] + source: heed::Error, + }, + #[error("Failed to start file system watcher: {0}")] + FileSystemWatch(#[from] notify::Error), + + #[error("Expected a path to be child of another path: {0}")] + StripPrefixError(#[from] StripPrefixError), + + #[error("libgit2 error occurred: {0}")] + Git(#[from] git2::Error), + + #[error("Filesystem walk failed: {0}")] + WalkFailed(String), +} + +pub type Result = std::result::Result; diff --git a/crates/fff-core/src/file_picker.rs b/crates/fff-core/src/file_picker.rs new file mode 100644 index 0000000..0d51745 --- /dev/null +++ b/crates/fff-core/src/file_picker.rs @@ -0,0 +1,2252 @@ +//! Core file picker: filesystem indexing, background watching, and fuzzy search. +//! +//! [`FilePicker`] is the central component of fff-search. It: +//! +//! 1. **Indexes** a directory tree in a background thread, collecting every +//! non-ignored file into a path-sorted `Vec`. +//! 2. **Watches** the filesystem via the `notify` crate, applying +//! create/modify/delete events to the index in real time. +//! 3. **Owns files**: Provides a values for search and provides a good entry point for +//! fuzzy search and live grep +//! +//! # Lifecycle +//! +//! ```text +//! new_with_shared_state() +//! │ +//! ├─> background scan thread ──> populates SharedPicker +//! └─> file-system watcher ──> live updates SharedPicker +//! +//! search() <── borrows &self, delegates to fuzzy_search +//! grep() <── static, borrows &[FileItem] (live content search) +//! trigger_rescan() <── synchronous re-index +//! cancel() <── shuts down background work +//! ``` +//! +//! # Thread Safety +//! +//! `FilePicker` itself is **not** `Sync`! +//! all concurrent access goes through [`SharedPicker`](crate::SharedPicker) . +//! The background scanner and watcher acquire write locks only when mutating +//! the file index, so read-heavy search workloads rarely contend. + +use crate::FFFStringStorage; +use crate::background_watcher::BackgroundWatcher; +use crate::bigram_filter::{BigramFilter, BigramOverlay}; +use crate::constants::{MAX_OVERFLOW_FILES, PATH_BUF_SIZE}; +use crate::error::Error; +use crate::frecency::FrecencyTracker; +use crate::git::GitStatusCache; +use crate::grep::{GrepResult, GrepSearchOptions, grep_search, multi_grep_search}; +use crate::query_tracker::QueryTracker; +use crate::scan::{ScanConfig, ScanJob, ScanSignals}; +use crate::score::fuzzy_match_and_score_files; +use crate::shared::{SharedFilePicker, SharedFrecency}; +use crate::simd_path::ArenaPtr; +use crate::stable_vec::StableVec; +use crate::types::{ + ContentCacheBudget, DirItem, DirSearchResult, FileItem, MixedItemRef, MixedSearchResult, + PaginationArgs, Score, ScoringContext, SearchResult, +}; +use fff_query_parser::FFFQuery; +use git2::{Repository, Status}; +use rayon::prelude::*; +use std::fmt::Debug; +use std::ops::ControlFlow; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use std::thread::JoinHandle; +use std::time::SystemTime; +use tracing::{Level, debug, error, info, warn}; + +use crate::parallelism::{BACKGROUND_THREAD_POOL, SEARCH_THREAD_POOL}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum FFFMode { + #[default] + Neovim, + Ai, +} + +impl FFFMode { + pub fn is_ai(self) -> bool { + self == FFFMode::Ai + } +} + +/// Configuration for a single fuzzy search invocation. +/// +/// Passed to [`FilePicker::search`] to control threading, pagination, +/// and scoring behavior. +#[derive(Debug, Clone, Copy, Default)] +pub struct FuzzySearchOptions<'a> { + pub max_threads: usize, + pub current_file: Option<&'a str>, + pub project_path: Option<&'a Path>, + pub combo_boost_score_multiplier: i32, + pub min_combo_count: u32, + pub pagination: PaginationArgs, +} + +#[derive(Debug, Clone)] +pub(crate) struct FileSync { + pub(crate) git_workdir: Option, + /// Base files laid out in two partitions, each internally sorted by + /// (parent_dir, filename): + /// `files[..indexable_count]` - indexable + /// `files[indexable_count..base_count]` - original-unindexable + /// `files[base_count..]`— overflow (created on demand) + files: StableVec, + indexable_count: usize, + base_count: usize, + /// Number of active present files that exists in the file system + pub(crate) live_count: usize, + /// Sorted directory table. `StableVec` so post-scan snapshots can keep + /// the allocation alive across a picker drop without copying, and so + /// concurrent readers observe a consistent view via the same shared + /// allocation. Dir frecency is updated through the per-entry atomic + /// (`DirItem::max_access_frecency`) without `&mut` aliasing. + dirs: StableVec, + /// Shared builder for overflow file paths. Each overflow file's ChunkedString + /// uses `arena_override` pointing into this builder's arena. + overflow_builder: Option, + bigram_index: Option>, + bigram_overlay: Option>>, + /// Chunk-level deduped path store. Arc so post-scan snapshots can hold + /// the arena alive while iterating file paths. + chunked_paths: Option>, + /// Ignore rules the walker assembled (zlob backend only). Shared with the + /// background watcher so filesystem events can be filtered without libgit2. + pub(crate) ignore_rules: Option>, +} + +impl FileSync { + fn new() -> Self { + Self { + files: StableVec::from_vec_with_reserve(Vec::new(), MAX_OVERFLOW_FILES), + indexable_count: 0, + base_count: 0, + live_count: 0, + dirs: StableVec::from_vec_with_reserve(Vec::new(), 0), + overflow_builder: None, + git_workdir: None, + bigram_index: None, + bigram_overlay: None, + chunked_paths: None, + ignore_rules: None, + } + } + + #[inline] + fn arena_base_ptr(&self) -> ArenaPtr { + self.chunked_paths + .as_ref() + .map(|s| s.as_arena_ptr()) + .unwrap_or(ArenaPtr::null()) + } + + #[inline] + fn arena_overflow_ptr(&self) -> ArenaPtr { + self.overflow_builder + .as_ref() + .map(|b| b.as_arena_ptr()) + .unwrap_or(ArenaPtr::null()) + } + + #[inline] + fn arena_for_file(&self, file: &FileItem) -> ArenaPtr { + if file.is_overflow() { + self.arena_overflow_ptr() + } else { + self.arena_base_ptr() + } + } + + #[inline] + fn files(&self) -> &[FileItem] { + &self.files + } + + #[inline] + fn overflow_files(&self) -> &[FileItem] { + &self.files[self.base_count..] + } + + #[inline] + fn get_file_mut(&mut self, index: usize) -> Option<(ArenaPtr, &mut FileItem)> { + Some(( + if index < self.base_count { + self.arena_base_ptr() + } else { + self.arena_overflow_ptr() + }, + self.files.get_mut(index)?, + )) + } + + #[inline] + fn find_file_index(&self, path: &Path, base_path: &Path) -> Option { + let arena = self.arena_base_ptr(); + + // Strip base_path prefix to get the relative path. On Windows this + // can fail for 8.3 short names or a different casing; fall back to + // canonicalize-then-strip so watcher events still land on the right + // `FileItem`. + let rel_path_owned: String = match path.strip_prefix(base_path) { + Ok(r) => r.to_string_lossy().into_owned(), + Err(_) => { + #[cfg(windows)] + { + canonical_relative_path(path, base_path)? + } + #[cfg(not(windows))] + { + return None; + } + } + }; + // The dir table and stored file paths are '/'-canonical; fold the + // native relative path so the byte-wise comparisons below match. + let rel_path_owned = crate::path_utils::to_canonical_slashes(&rel_path_owned).into_owned(); + let rel_path: &str = &rel_path_owned; + + // Split into directory (with trailing '/') and filename. + let parent_end = rel_path + .rfind(std::path::is_separator) + .map(|i| i + 1) + .unwrap_or(0); + let dir_rel = &rel_path[..parent_end]; + let filename = &rel_path[parent_end..]; + + // Binary search dirs to find the parent directory index. + // Dir items store the relative path including trailing '/' (e.g. "src/components/"). + let mut dir_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + let dir_idx = self + .dirs + .binary_search_by(|d| d.read_relative_path(arena, &mut dir_buf).cmp(dir_rel)) + .ok(); + + // Binary search base files by (parent_dir, filename). Base files live in + // two internally-sorted partitions — indexable first, then unindexable — + // so we try each half in turn. Two O(log n) searches with short-circuit. + if let Some(dir_idx) = dir_idx { + let dir_idx = dir_idx as u32; + let cmp_key = |f: &FileItem| { + f.parent_dir_index.cmp(&dir_idx).then_with(|| { + let fname = f.file_name(arena); + fname.as_str().cmp(filename) + }) + }; + + if self.indexable_count > 0 + && let Ok(pos) = self.files[..self.indexable_count].binary_search_by(cmp_key) + { + return Some(pos); + } + + if self.indexable_count < self.base_count + && let Ok(rel_pos) = + self.files[self.indexable_count..self.base_count].binary_search_by(cmp_key) + { + return Some(self.indexable_count + rel_pos); + } + } + + // Overflow region: linear scan by full relative path. + if self.base_count < self.files.len() { + let overflow_arena = self.arena_overflow_ptr(); + if let Some(pos) = self.files[self.base_count..] + .iter() + .position(|f| f.relative_path_eq(overflow_arena, rel_path)) + { + return Some(self.base_count + pos); + } + } + + None + } + + // TODO remove this function and make a better way to remove all files + // from the directory without looping over the whole sync data list + /// Tombstones every file in the arena that matches certain predicate + fn tombstone_files_with_arena(&mut self, mut predicate: F) -> usize + where + F: FnMut(&FileItem, ArenaPtr) -> bool, + { + let base_arena = self.arena_base_ptr(); + let overflow_arena = self.arena_overflow_ptr(); + let base_count = self.base_count; + + let mut tombstoned = 0usize; + for (idx, file) in self.files.iter_mut().enumerate() { + if file.is_deleted() { + continue; + } + let arena = if idx < base_count { + base_arena + } else { + overflow_arena + }; + if predicate(file, arena) { + file.set_deleted(true); + tombstoned += 1; + } + } + self.live_count -= tombstoned; + tombstoned + } +} + +impl FileItem { + pub fn new(path: PathBuf, base_path: &Path, git_status: Option) -> (Self, String) { + let metadata = std::fs::metadata(&path).ok(); + Self::new_with_metadata(path, base_path, git_status, metadata.as_ref()) + } + + /// Create a FileItem using pre-fetched metadata to avoid a redundant stat syscall. + /// Returns `(FileItem, relative_path)`. The FileItem's `path` field is + /// empty; callers must populate it via `set_path` or `build_chunked_path_store_and_assign`. + fn new_with_metadata( + path: PathBuf, + base_path: &Path, + git_status: Option, + metadata: Option<&std::fs::Metadata>, + ) -> (Self, String) { + let path_buf = pathdiff::diff_paths(&path, base_path).unwrap_or_else(|| path.clone()); + // The index is '/'-canonical on every platform; fold native separators. + let relative_path = + crate::path_utils::to_canonical_slashes(&path_buf.to_string_lossy()).into_owned(); + + let (size, modified) = match metadata { + Some(metadata) => { + let size = metadata.len(); + let modified = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map_or(0, |d| d.as_secs()); + + (size, modified) + } + None => (0, 0), + }; + + let is_binary = is_known_binary_extension(&path); + + let filename_start = relative_path + .rfind(std::path::is_separator) + .map(|i| i + 1) + .unwrap_or(0) as u16; + + let item = Self::new_raw(filename_start, size, modified, git_status, is_binary); + (item, relative_path) + } + + /// Create a FileItem with an empty ChunkedString from a path on disk. + /// + /// Returns `(file_item, relative_path_string)`. The relative path must be + /// kept alongside the FileItem until `build_chunked_path_store_and_assign` + /// populates each item's `path` field from the shared arena. + pub fn new_from_walk( + path: &Path, + base_path: &Path, + git_status: Option, + metadata: Option<&std::fs::Metadata>, + ) -> (Self, String) { + let (size, modified) = match metadata { + Some(metadata) => { + let size = metadata.len(); + let modified = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map_or(0, |d| d.as_secs()); + (size, modified) + } + None => (0, 0), + }; + + Self::new_from_walk_parts(path, base_path, git_status, size, modified) + } + + /// Like [`Self::new_from_walk`] but takes already-extracted size and + /// modification time (Unix seconds) instead of a `std::fs::Metadata`. + /// Used by the zlob walker backend, which fetches metadata in bulk. + pub fn new_from_walk_parts( + path: &Path, + base_path: &Path, + git_status: Option, + size: u64, + modified: u64, + ) -> (Self, String) { + let is_binary = is_known_binary_extension(path); + + let rel = pathdiff::diff_paths(path, base_path).unwrap_or_else(|| path.to_path_buf()); + // The index is '/'-canonical on every platform; fold native separators. + let rel_str = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy()).into_owned(); + let fname_offset = rel_str + .rfind(std::path::is_separator) + .map(|i| i + 1) + .unwrap_or(0) as u16; + + let item = Self::new_raw(fname_offset, size, modified, git_status, is_binary); + (item, rel_str) + } + + /// Zlob-walker fast path: skip the `pathdiff::diff_paths` PathBuf alloc by + /// taking the already-relative slice and the basename-offset that zlob's + /// scanner computed during traversal. ~80–120 ms saved on a chromium scan + /// (500k entries × one fewer alloc + no component walk). + /// + /// `relative_path` is root-relative bytes; `basename_offset` is the byte + /// offset where the basename begins (e.g. zlob's `entry.path_bytes().len() + /// - entry.file_name().as_os_str().as_encoded_bytes().len()` minus the + /// `relative_offset`). + pub fn new_from_walk_bytes( + path: &Path, + relative_path: &[u8], + basename_offset: u16, + git_status: Option, + size: u64, + modified: u64, + ) -> (Self, String) { + let is_binary = is_known_binary_extension(path); + // SAFETY-ish: paths on macOS/Linux are bytes; lossy conversion mirrors + // the existing `to_string_lossy()` behavior on non-UTF8 names. + let rel_str = String::from_utf8_lossy(relative_path).into_owned(); + let item = Self::new_raw(basename_offset, size, modified, git_status, is_binary); + (item, rel_str) + } + + pub(crate) fn update_frecency_scores( + &mut self, + tracker: &FrecencyTracker, + arena: ArenaPtr, + base_path: &Path, + mode: FFFMode, + ) -> Result<(), Error> { + let mut abs_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + let abs = self.write_absolute_path(arena, base_path, &mut abs_buf); + self.access_frecency_score = tracker.get_access_score(abs, mode) as i16; + self.modification_frecency_score = + tracker.get_modification_score(self.modified, self.git_status, mode) as i16; + + Ok(()) + } +} + +/// Options for creating a [`FilePicker`]. +pub struct FilePickerOptions { + pub base_path: String, + /// Pre-populate mmap caches for top-frecency files after the initial scan. + pub enable_mmap_cache: bool, + /// Build content index after the initial scan for faster content-aware filtering. + pub enable_content_indexing: bool, + /// Mode of the picker impact the way file watcher events are handled and the scoring logic + pub mode: FFFMode, + /// Explicit cache budget. When `None`, the budget is auto-computed from + /// the repo size after the initial scan completes. + pub cache_budget: Option, + /// When `false`, `new_with_shared_state` skips the background file watcher. + pub watch: bool, + /// Follow symbolic links during file indexing. + pub follow_symlinks: bool, + /// Allow indexing the filesystem root (`/`). Off by default — these dirs + /// generate enormous fs-event traffic and are rarely the intended target. + pub enable_fs_root_scanning: bool, + /// Allow indexing the user's home directory. Off by default for the same + /// reason as `enable_fs_root_scanning`. + pub enable_home_dir_scanning: bool, +} + +impl Default for FilePickerOptions { + fn default() -> Self { + Self { + base_path: ".".into(), + enable_mmap_cache: false, + enable_content_indexing: false, + mode: FFFMode::default(), + cache_budget: None, + watch: true, + follow_symlinks: false, + enable_fs_root_scanning: false, + enable_home_dir_scanning: false, + } + } +} + +pub struct FilePicker { + pub mode: FFFMode, + pub base_path: PathBuf, + sync_data: FileSync, + pub(crate) signals: ScanSignals, + pub(crate) background_watcher: Option, + /// Single serialized writer for all git-status updates (scan, watcher, + /// FFI). Owned by the picker so it exists before the first scan; its + /// consumer thread is spawned lazily once a git workdir is discovered. + pub(crate) git_status_worker: Arc, + cache_budget: Arc, + has_explicit_cache_budget: bool, + scanned_files_count: Arc, + enable_mmap_cache: bool, + enable_content_indexing: bool, + watch: bool, + follow_symlinks: bool, + enable_fs_root_scanning: bool, + enable_home_dir_scanning: bool, + trace_span: tracing::Span, + trace_id: String, +} + +impl std::fmt::Debug for FilePicker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FilePicker") + .field("base_path", &self.base_path) + .field("sync_data", &self.sync_data) + .field( + "is_scanning", + &self.signals.scanning.load(Ordering::Relaxed), + ) + .field( + "scanned_files_count", + &self.scanned_files_count.load(Ordering::Relaxed), + ) + .finish_non_exhaustive() + } +} + +impl FFFStringStorage for &FilePicker { + #[inline] + fn arena_for(&self, file: &FileItem) -> crate::simd_path::ArenaPtr { + self.sync_data.arena_for_file(file) + } + + #[inline] + fn base_arena(&self) -> crate::simd_path::ArenaPtr { + self.sync_data.arena_base_ptr() + } + + #[inline] + fn overflow_arena(&self) -> crate::simd_path::ArenaPtr { + self.sync_data.arena_overflow_ptr() + } +} + +impl FilePicker { + pub fn base_path(&self) -> &Path { + &self.base_path + } + + /// Ignore rules the walker assembled during the last scan (zlob backend + /// only). The background watcher uses these to filter events without + /// libgit2. `None` when the backend doesn't surface rules or no ignore + /// files were present. + pub(crate) fn ignore_rules(&self) -> Option> { + self.sync_data.ignore_rules.clone() + } + + pub fn has_mmap_cache(&self) -> bool { + self.enable_mmap_cache + } + + pub fn has_content_indexing(&self) -> bool { + self.enable_content_indexing + } + + pub fn has_watcher(&self) -> bool { + self.watch + } + + pub fn follows_symlinks(&self) -> bool { + self.follow_symlinks + } + + pub fn fs_root_scanning_enabled(&self) -> bool { + self.enable_fs_root_scanning + } + + pub fn home_dir_scanning_enabled(&self) -> bool { + self.enable_home_dir_scanning + } + + pub fn trace_id(&self) -> &str { + &self.trace_id + } + + pub fn trace_span(&self) -> tracing::Span { + self.trace_span.clone() + } + + pub fn mode(&self) -> FFFMode { + self.mode + } + + pub fn cache_budget(&self) -> &ContentCacheBudget { + &self.cache_budget + } + + pub fn bigram_index(&self) -> Option<&BigramFilter> { + self.sync_data.bigram_index.as_deref() + } + + pub fn bigram_overlay(&self) -> Option<&parking_lot::RwLock> { + self.sync_data.bigram_overlay.as_deref() + } + + pub fn get_file_mut(&mut self, index: usize) -> Option<(ArenaPtr, &mut FileItem)> { + self.sync_data.get_file_mut(index) + } + + /// Absolute path to the repository root if the indexed tree lives + /// inside a git working directory. `None` for non-git bases. + pub fn git_root(&self) -> Option<&Path> { + self.sync_data.git_workdir.as_deref() + } + + pub fn has_explicit_cache_budget(&self) -> bool { + self.has_explicit_cache_budget + } + + pub fn set_cache_budget(&mut self, budget: ContentCacheBudget) { + self.cache_budget = Arc::new(budget); + } + + /// Get all indexed files sorted by path. + /// Note: Files are stored sorted by PATH for efficient insert/remove. + /// For frecency-sorted results, use search() which sorts matched results. + pub fn get_files(&self) -> &[FileItem] { + self.sync_data.files() + } + + /// Count of live (non-tombstoned) files. O(1). + #[inline] + pub fn live_file_count(&self) -> usize { + self.sync_data.live_count + } + + pub fn get_overflow_files(&self) -> &[FileItem] { + self.sync_data.overflow_files() + } + + /// Get the directory table (sorted by path). + pub fn get_dirs(&self) -> &[DirItem] { + &self.sync_data.dirs + } + + /// Actual heap bytes used: (chunked_path_store, 0, 0). + /// The second element is 0 because leaked overflow stores aren't tracked. + pub fn arena_bytes(&self) -> (usize, usize, usize) { + let chunked = self + .sync_data + .chunked_paths + .as_ref() + .map_or(0, |s| s.heap_bytes()); + + (chunked, 0, 0) + } + + #[tracing::instrument(level = "debug", skip_all)] + pub(crate) fn for_each_dir(&self, mut f: impl FnMut(&Path) -> ControlFlow<()>) { + let dir_table = &self.sync_data.dirs; + let base = self.base_path.as_path(); + + if !dir_table.is_empty() { + let arena = self.arena_base_ptr(); + let mut path_buf = PathBuf::with_capacity(crate::simd_path::PATH_BUF_SIZE); + let mut prev_relative_path = String::new(); + + let mut scratch_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + for dir_item in dir_table.iter() { + let full_relative_path = dir_item.read_relative_path(arena, &mut scratch_buf); + let relative_path = full_relative_path.trim_end_matches(std::path::is_separator); + + if relative_path.is_empty() { + // Files directly under base_path + prev_relative_path.clear(); + continue; + } + + let mut i = common_dir_prefix_len(&prev_relative_path, relative_path); + // If we stopped on a separator, skip it — we want to start + // emitting at the first unseen segment, not re-emit the + // already-emitted prefix path. + if i < relative_path.len() + && std::path::is_separator(relative_path.as_bytes()[i] as char) + { + i += 1; + } + + // Walk the suffix of `relative_path` one segment at a time, emitting + // each previously unseen ancestor up to and including `relative_path`. + while i < relative_path.len() { + let next_sep = relative_path[i..] + .find(std::path::is_separator) + .map(|off| i + off) + .unwrap_or(relative_path.len()); + let ancestor_rel = &relative_path[..next_sep]; + + path_buf.clear(); + path_buf.push(base); + path_buf.push(ancestor_rel); + + // we can't really emit iterator here unfortunately + if matches!(f(path_buf.as_path()), ControlFlow::Break(())) { + return; + } + + i = next_sep + 1; + } + + prev_relative_path.clear(); + prev_relative_path.push_str(relative_path); + } + return; + } + + // fallback that should never be happening, but it is possible to get the file + // path from the absolute path using components api as well: + let files = self.sync_data.files(); + let arena = self.arena_base_ptr(); + let mut current = self.base_path.clone(); + let mut path_buf = [0u8; PATH_BUF_SIZE]; + + for file in files { + let abs = file.write_absolute_path(arena, base, &mut path_buf); + let Some(parent) = abs.parent() else { + continue; + }; + if parent == current.as_path() { + continue; + } + + while current.as_path() != base && !parent.starts_with(¤t) { + current.pop(); + } + + let Ok(remainder) = parent.strip_prefix(¤t) else { + continue; + }; + for component in remainder.components() { + current.push(component); + if matches!(f(current.as_path()), ControlFlow::Break(())) { + return; + } + } + } + } + + /// Create a new FilePicker from options. + /// Always prefer new_with_shared_state for the consumer application, use this only if you know + /// what you are doing. This won't spawn the backgraound watcher and won't walk the file tree. + pub fn new(options: FilePickerOptions) -> Result { + let path = PathBuf::from(&options.base_path); + if !path.exists() { + error!("Base path does not exist: {}", options.base_path); + return Err(Error::InvalidPath(path)); + } + if path.parent().is_none() && !options.enable_fs_root_scanning { + error!("Refusing to index filesystem root: {}", path.display()); + return Err(Error::FilesystemRoot(path)); + } + if !options.enable_home_dir_scanning + && Some(path.as_os_str()) == dirs::home_dir().as_ref().map(|p| p.as_os_str()) + { + error!("Refusing to index home directory: {}", path.display()); + return Err(Error::FilesystemRoot(path)); + } + + // Windows-only: canonicalize with so the base path does NOT + // have the `\\?\` UNC prefix that `std::fs::canonicalize` adds. + // libgit2's `repo.workdir()` + #[cfg(windows)] + let path = crate::path_utils::canonicalize(&path).unwrap_or(path); + + let has_explicit_budget = options.cache_budget.is_some(); + let initial_budget = options.cache_budget.unwrap_or_default(); + + let trace_id = crate::log::generate_trace_id(); + let trace_span = crate::log::trace_span(&trace_id, "picker"); + + Ok(FilePicker { + background_watcher: None, + git_status_worker: crate::git_status_worker::GitStatusWorker::new(), + base_path: path, + cache_budget: Arc::new(initial_budget), + has_explicit_cache_budget: has_explicit_budget, + signals: crate::scan::ScanSignals::default(), + mode: options.mode, + scanned_files_count: Arc::new(AtomicUsize::new(0)), + sync_data: FileSync::new(), + enable_mmap_cache: options.enable_mmap_cache, + enable_content_indexing: options.enable_content_indexing, + watch: options.watch, + follow_symlinks: options.follow_symlinks, + enable_fs_root_scanning: options.enable_fs_root_scanning, + enable_home_dir_scanning: options.enable_home_dir_scanning, + trace_span, + trace_id, + }) + } + + /// Create a picker, place it into the shared handle, and spawn background + /// indexing + file-system watcgenerate_trace_id the default entry point. + pub fn new_with_shared_state( + shared_picker: SharedFilePicker, + shared_frecency: SharedFrecency, + options: FilePickerOptions, + ) -> Result<(), Error> { + let picker = Self::new(options)?; + + info!( + "Spawning background threads: base_path={}, warmup={}, content_indexing={}, mode={:?}", + picker.base_path.display(), + picker.enable_mmap_cache, + picker.enable_content_indexing, + picker.mode, + ); + + let warmup = picker.enable_mmap_cache; + let content_indexing = picker.enable_content_indexing; + let watch = picker.watch; + let mode = picker.mode; + let follow_symlinks = picker.follow_symlinks; + let enable_fs_root_scanning = picker.enable_fs_root_scanning; + let enable_home_dir_scanning = picker.enable_home_dir_scanning; + + let signals = picker.scan_signals(); + let scanned_files_counter = picker.scanned_files_counter(); + let path = picker.base_path.clone(); + let trace_span = picker.trace_span.clone(); + + // Pre-arm `scanning` BEFORE publishing the new picker. `ScanJob::spawn` + // also sets it, but that runs after this function returns; consumers + // (e.g. lua `wait_for_initial_scan` after `restart_index_in_path`) + // that grab the signal Arc between publish and spawn would otherwise + // observe scanning=false and skip the wait, racing the walker. The + // race is wide on Windows CI where notify is slow. + signals + .scanning + .store(true, std::sync::atomic::Ordering::Release); + + { + let mut guard = shared_picker.write()?; + *guard = Some(picker); + // dropping old picker flips its `cancelled` flag → bg threads exit cleanly + } + + ScanJob::new_initial( + shared_picker, + shared_frecency, + path, + mode, + signals, + scanned_files_counter, + trace_span, + ScanConfig { + warmup, + content_indexing, + watch, + auto_cache_budget: true, + install_watcher: true, + follow_symlinks, + enable_fs_root_scanning, + enable_home_dir_scanning, + }, + ) + .spawn(); + + Ok(()) + } + + /// Synchronous filesystem scan — populates `self` with indexed files. + /// + /// Use this when you need direct access to the picker without shared state: + /// ```ignore + /// let mut picker = FilePicker::new(options)?; + /// picker.collect_files()?; + /// // picker.get_files() is now populated + /// ``` + pub fn collect_files(&mut self) -> Result<(), Error> { + self.signals.scanning.store(true, Ordering::Relaxed); + self.scanned_files_count.store(0, Ordering::Relaxed); + + let git_workdir = FileSync::discover_git_workdir(&self.base_path); + let git_handle = git_workdir.clone().map(FileSync::spawn_git_status); + + let empty_frecency = SharedFrecency::default(); + let sync = FileSync::walk_filesystem( + &self.base_path, + git_workdir, + &self.scanned_files_count, + &empty_frecency, + self.mode, + self.follow_symlinks, + )?; + + self.sync_data = sync; + + if !self.has_explicit_cache_budget { + let file_count = self.sync_data.files().len(); + self.cache_budget = Arc::new(ContentCacheBudget::new_for_repo(file_count)); + } else { + self.cache_budget.reset(); + } + + if let Some(handle) = git_handle + && let Ok(Some(git_cache)) = handle.join() + { + let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + + let arena = self.arena_base_ptr(); + for file in self.sync_data.files.iter_mut() { + file.git_status = git_cache.lookup_status(file.write_absolute_path( + arena, + &self.base_path, + &mut path_buf, + )); + } + } + + self.signals.scanning.store(false, Ordering::Relaxed); + Ok(()) + } + + /// Perform fuzzy search on files with a pre-parsed query. + /// + /// The query should be parsed using [`crate::FFFQuery`] before calling + /// this function. If a [`crate::QueryTracker`] is provided, the search will + /// automatically look up the last selected file for this query and boost it + #[tracing::instrument(skip_all, name = "Fuzzy file search", fields(query = query.raw_query))] + pub fn fuzzy_search<'q>( + &self, + query: &'q FFFQuery<'q>, + query_tracker: Option<&QueryTracker>, + options: FuzzySearchOptions<'q>, + ) -> SearchResult<'_> { + let files = self.get_files(); + let max_threads = if options.max_threads == 0 { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + } else { + options.max_threads + }; + + debug!( + raw_query = ?query.raw_query, + pagination = ?options.pagination, + ?max_threads, + current_file = ?options.current_file, + "Fuzzy search", + ); + + let total_files = self.live_file_count(); + let location = query.location; + + // Get effective query for max_typos calculation (without location suffix) + let effective_query = match &query.fuzzy_query { + fff_query_parser::FuzzyQuery::Text(t) => *t, + fff_query_parser::FuzzyQuery::Parts(parts) if !parts.is_empty() => parts[0], + _ => query.raw_query.trim(), + }; + + // small queries with a large number of results can match absolutely everything + let max_typos = (effective_query.len() as u16 / 4).clamp(2, 6); + // Look up the last file selected for this query (combo-boost scoring) + let last_same_query_entry = + query_tracker + .zip(options.project_path) + .and_then(|(tracker, project_path)| { + tracker + .get_last_query_entry( + query.raw_query, + project_path, + options.min_combo_count, + ) + .ok() + .flatten() + }); + + let context = ScoringContext { + query, + max_typos, + max_threads, + project_path: options.project_path, + current_file: options.current_file, + last_same_query_match: last_same_query_entry, + combo_boost_score_multiplier: options.combo_boost_score_multiplier, + min_combo_count: options.min_combo_count, + pagination: options.pagination, + }; + + let time = std::time::Instant::now(); + + let base_arena = self.sync_data.arena_base_ptr(); + let overflow_arena = self.sync_data.arena_overflow_ptr(); + + let (items, scores, total_matched) = fuzzy_match_and_score_files( + files, + &context, + self.sync_data.base_count, + base_arena, + overflow_arena, + ); + + info!( + ?query, + completed_in = ?time.elapsed(), + total_matched, + returned_count = items.len(), + pagination = ?options.pagination, + "Fuzzy search completed", + ); + + SearchResult { + items, + scores, + total_matched, + total_files, + location, + } + } + + /// Perform fuzzy search on indexed directories. + /// + /// Returns directories ranked by fuzzy match quality + frecency. + pub fn fuzzy_search_directories<'q>( + &self, + query: &'q FFFQuery<'q>, + options: FuzzySearchOptions<'q>, + ) -> DirSearchResult<'_> { + let dirs = self.get_dirs(); + let max_threads = if options.max_threads == 0 { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + } else { + options.max_threads + }; + + let total_dirs = dirs.len(); + + let effective_query = match &query.fuzzy_query { + fff_query_parser::FuzzyQuery::Text(t) => *t, + fff_query_parser::FuzzyQuery::Parts(parts) if !parts.is_empty() => parts[0], + _ => query.raw_query.trim(), + }; + + let max_typos = (effective_query.len() as u16 / 4).clamp(2, 6); + + let context = ScoringContext { + query, + max_typos, + max_threads, + project_path: options.project_path, + current_file: options.current_file, + last_same_query_match: None, + combo_boost_score_multiplier: 0, + min_combo_count: 0, + pagination: options.pagination, + }; + + let arena = self.sync_data.arena_base_ptr(); + let time = std::time::Instant::now(); + + let (items, scores, total_matched) = + crate::score::fuzzy_match_and_score_dirs(dirs, &context, arena); + + info!( + ?query, + completed_in = ?time.elapsed(), + total_matched, + returned_count = items.len(), + "Directory search completed", + ); + + DirSearchResult { + items, + scores, + total_matched, + total_dirs, + } + } + + /// Perform a mixed fuzzy search across both files and directories. + /// + /// Returns a single flat list where files and directories are interleaved + /// by total score in descending order. + /// + /// If the raw query ends with a path separator (`/`), only directories + /// are searched — files are skipped entirely. The caller should parse the + /// query with `DirSearchConfig` so that trailing `/` is kept as fuzzy + /// text instead of becoming a `PathSegment` constraint. + pub fn fuzzy_search_mixed<'q>( + &self, + query: &'q FFFQuery<'q>, + query_tracker: Option<&QueryTracker>, + options: FuzzySearchOptions<'q>, + ) -> MixedSearchResult<'_> { + let location = query.location; + let page_offset = options.pagination.offset; + let page_limit = if options.pagination.limit > 0 { + options.pagination.limit + } else { + 100 + }; + + let dirs_only = + query.raw_query.ends_with(std::path::MAIN_SEPARATOR) || query.raw_query.ends_with('/'); + + // Run file search and dir search with no pagination (we merge then paginate). + let internal_limit = page_offset.saturating_add(page_limit).saturating_mul(2); + + let dir_options = FuzzySearchOptions { + pagination: PaginationArgs { + offset: 0, + limit: internal_limit, + }, + ..options + }; + let dir_results = self.fuzzy_search_directories(query, dir_options); + + if dirs_only { + let total_matched = dir_results.total_matched; + let total_dirs = dir_results.total_dirs; + + let mut merged: Vec<(MixedItemRef<'_>, Score)> = + Vec::with_capacity(dir_results.items.len()); + for (dir, score) in dir_results.items.into_iter().zip(dir_results.scores) { + merged.push((MixedItemRef::Dir(dir), score)); + } + + if page_offset >= merged.len() { + return MixedSearchResult { + items: vec![], + scores: vec![], + total_matched, + total_files: self.live_file_count(), + total_dirs, + location, + }; + } + + let end = (page_offset + page_limit).min(merged.len()); + let page = merged.drain(page_offset..end); + let (items, scores): (Vec<_>, Vec<_>) = page.unzip(); + + return MixedSearchResult { + items, + scores, + total_matched, + total_files: self.live_file_count(), + total_dirs, + location, + }; + } + + let file_options = FuzzySearchOptions { + pagination: PaginationArgs { + offset: 0, + limit: internal_limit, + }, + ..options + }; + let file_results = self.fuzzy_search(query, query_tracker, file_options); + + // Merge by score descending. + let total_matched = file_results.total_matched + dir_results.total_matched; + let total_files = file_results.total_files; + let total_dirs = dir_results.total_dirs; + + let mut merged: Vec<(MixedItemRef<'_>, Score)> = + Vec::with_capacity(file_results.items.len() + dir_results.items.len()); + + for (file, score) in file_results.items.into_iter().zip(file_results.scores) { + merged.push((MixedItemRef::File(file), score)); + } + for (dir, score) in dir_results.items.into_iter().zip(dir_results.scores) { + merged.push((MixedItemRef::Dir(dir), score)); + } + + // Sort merged results by total score descending. + merged.sort_unstable_by_key(|b| std::cmp::Reverse(b.1.total)); + + // Paginate. + if page_offset >= merged.len() { + return MixedSearchResult { + items: vec![], + scores: vec![], + total_matched, + total_files, + total_dirs, + location, + }; + } + + let end = (page_offset + page_limit).min(merged.len()); + let page = merged.drain(page_offset..end); + let (items, scores): (Vec<_>, Vec<_>) = page.unzip(); + + MixedSearchResult { + items, + scores, + total_matched, + total_files, + total_dirs, + location, + } + } + + /// Glob search: filter indexed files by a single glob pattern, rank by + /// frecency, and paginate. Bypasses the regular query parser entirely — + /// useful when callers already have a literal glob (`*.rs`, `**/*.test.ts`) + /// and want neither fuzzy matching nor multi-token constraint parsing. + /// + /// Pipeline: `apply_constraints(Glob) → score_filtered_by_frecency → sort_and_paginate`. + /// Same ranking semantics as `fuzzy_search` when the fuzzy query is empty. + pub fn glob<'p>( + &'p self, + pattern: &'p str, + options: FuzzySearchOptions<'p>, + ) -> SearchResult<'p> { + let query = FFFQuery { + raw_query: pattern, + constraints: vec![fff_query_parser::Constraint::Glob(pattern)], + fuzzy_query: fff_query_parser::FuzzyQuery::Empty, + location: None, + }; + + // `fuzzy_search` short-circuits to `score_filtered_by_frecency` when + // `fuzzy_query` is `Empty`, then runs the same `sort_and_paginate` + // path. Reusing it keeps the ranking guarantees identical without + // exposing the private scoring helpers. + self.fuzzy_search(&query, None, options) + } + + /// Perform a live grep search across indexed files. + /// + /// If `options.abort_signal` is set it overrides the picker's internal + /// cancellation flag, giving the caller full control over when to stop. + pub fn grep(&self, query: &FFFQuery<'_>, options: &GrepSearchOptions) -> GrepResult<'_> { + let overlay_guard = self.sync_data.bigram_overlay.as_ref().map(|o| o.read()); + let arena = self.arena_base_ptr(); + let overflow_arena = self.sync_data.arena_overflow_ptr(); + let cancel = options + .abort_signal + .as_deref() + .unwrap_or(&self.signals.cancelled); + + SEARCH_THREAD_POOL.install(|| { + grep_search( + self.get_files(), + query, + options, + self.cache_budget(), + self.sync_data.bigram_index.as_deref(), + overlay_guard.as_deref(), + cancel, + &self.base_path, + arena, + overflow_arena, + ) + }) + } + + /// Multi-pattern grep search across indexed files. + pub fn multi_grep( + &self, + patterns: &[&str], + constraints: &[fff_query_parser::Constraint<'_>], + options: &GrepSearchOptions, + ) -> GrepResult<'_> { + let overlay_guard = self.sync_data.bigram_overlay.as_ref().map(|o| o.read()); + let arena = self.arena_base_ptr(); + let overflow_arena = self.sync_data.arena_overflow_ptr(); + let cancel = options + .abort_signal + .as_deref() + .unwrap_or(&self.signals.cancelled); + + SEARCH_THREAD_POOL.install(|| { + multi_grep_search( + self.get_files(), + patterns, + constraints, + options, + self.cache_budget(), + self.sync_data.bigram_index.as_deref(), + overlay_guard.as_deref(), + cancel, + &self.base_path, + arena, + overflow_arena, + ) + }) + } + + // Returns an ongoing or finisshed scan progress + pub fn get_scan_progress(&self) -> ScanProgress { + let scanned_count = self.scanned_files_count.load(Ordering::Relaxed); + let is_scanning = self.signals.scanning.load(Ordering::Relaxed); + + ScanProgress { + scanned_files_count: scanned_count, + is_scanning, + is_watcher_ready: self.signals.watcher_ready.load(Ordering::Relaxed), + is_warmup_complete: !self.enable_content_indexing + || self.sync_data.bigram_index.is_some(), + } + } + + pub(crate) fn set_bigram_index(&mut self, index: BigramFilter) { + self.sync_data.bigram_index = Some(Arc::new(index)); + // once the index is reset automatically reset the overaly + self.sync_data.bigram_overlay = Some(Arc::new(parking_lot::RwLock::new( + BigramOverlay::new(self.sync_data.indexable_count), + ))); + } + + pub(crate) fn scan_signals(&self) -> crate::scan::ScanSignals { + self.signals.clone() + } + + pub(crate) fn scanned_files_counter(&self) -> Arc { + Arc::clone(&self.scanned_files_count) + } + + /// Capture raw pointers to the picker's internal arrays for off-lock use. + /// + /// Sets `post_scan_indexing_active` and returns a snapshot that clears it + /// on drop. This is the ONLY approved way to escape the lock for + /// long-running parallel work (git status, warmup, bigram). + /// + /// Returns `None` if `post_scan_indexing_active` is already set — this + /// means another post-scan is in flight and we must not create a second + /// set of dangling pointers. + /// + /// # Safety + /// 1. `walk_filesystem` reserved `MAX_OVERFLOW_FILES` capacity on the + /// files Vec at creation — watcher pushes cannot reallocate it. + /// 2. `post_scan_indexing_active` is set — prevents `commit_new_sync` + /// from replacing the Vec (ScanJob::new checks this flag). + /// 3. Only `[..base_count]` is accessed — base files use the immutable + /// base arena. Overflow files use a different arena. + pub(crate) unsafe fn post_scan_snapshot(&self) -> Option { + if self + .signals + .post_scan_indexing_active + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + tracing::error!( + "Can not acquire post scan unsafe snapshot, someone already acquired it" + ); + return None; + } + + Some(PostScanUnsafeSnapshot { + files: self.sync_data.files.clone(), + arena: self.sync_data.chunked_paths.as_ref().map(Arc::clone), + base_count: self.sync_data.base_count, + indexable_count: self.sync_data.indexable_count, + base_path: self.base_path.clone(), + post_scan_flag: Arc::clone(&self.signals.post_scan_indexing_active), + _budget: Arc::clone(&self.cache_budget), + }) + } + + pub(crate) fn commit_new_sync(&mut self, sync: FileSync) { + self.sync_data = sync; + self.cache_budget.reset(); + } + + #[inline] + pub(crate) fn arena_base_ptr(&self) -> ArenaPtr { + self.sync_data.arena_base_ptr() + } + + /// Update git statuses for files, using the provided shared frecency tracker. + pub(crate) fn update_git_statuses( + &mut self, + status_cache: GitStatusCache, + shared_frecency: &SharedFrecency, + ) -> Result<(), Error> { + debug!( + statuses_count = status_cache.statuses_len(), + "Updating git status", + ); + + let mode = self.mode; + let bp = self.base_path.clone(); + let frecency = shared_frecency.read()?; + + status_cache + .into_iter() + .try_for_each(|(path, status)| -> Result<(), Error> { + if let Some((arena, file)) = self.get_mut_file_by_path(&path) { + file.git_status = Some(status); + if let Some(ref f) = *frecency { + file.update_frecency_scores(f, arena, &bp, mode)?; + } + // Update parent dir frecency inline. `DirItem` has an + // interior-mutable atomic score, so `&self` access is + // enough — no write aliasing against Arc clones. + let score = file.access_frecency_score as i32; + let dir_idx = file.parent_dir_index as usize; + if let Some(dir) = self.sync_data.dirs.get(dir_idx) { + dir.update_frecency_if_larger(score); + } + } else { + // Expected on sparse checkouts: git reports a status for + // a path that isn't materialized on disk and therefore + // isn't in the file index. Don't spam the log (#404). + debug!(?path, "Git status for path not in index, skipping"); + } + Ok(()) + })?; + + Ok(()) + } + + pub fn update_single_file_frecency( + &mut self, + file_path: impl AsRef, + frecency_tracker: &FrecencyTracker, + ) -> Result<(), Error> { + let path = file_path.as_ref(); + + let Some(index) = self.sync_data.find_file_index(path, &self.base_path) else { + return Ok(()); + }; + + if let Some((arena, file)) = self.sync_data.get_file_mut(index) { + file.update_frecency_scores(frecency_tracker, arena, &self.base_path, self.mode)?; + + // Update parent dir frecency inline (atomic, &self access). + let score = file.access_frecency_score as i32; + let dir_idx = file.parent_dir_index as usize; + if let Some(dir) = self.sync_data.dirs.get(dir_idx) { + dir.update_frecency_if_larger(score); + } + } + + Ok(()) + } + + pub fn get_file_by_path(&self, path: impl AsRef) -> Option<&FileItem> { + self.sync_data + .find_file_index(path.as_ref(), &self.base_path) + .and_then(|index| self.sync_data.files().get(index)) + } + + pub fn get_mut_file_by_path( + &mut self, + path: impl AsRef, + ) -> Option<(ArenaPtr, &mut FileItem)> { + let path = path.as_ref(); + let index = self.sync_data.find_file_index(path, &self.base_path); + index.and_then(|i| self.sync_data.get_file_mut(i)) + } + + /// Handle the event of certain file being modified or adds a neww file if it is not added + /// If this function returns `None` it means that picker is in the invalid state, or the capacity + /// of index is exhausted and a new rescan needs to be triggered. + #[tracing::instrument(skip(self),level = Level::DEBUG)] + pub fn handle_create_or_modify(&mut self, path: impl AsRef + Debug) -> Option<&FileItem> { + let path = path.as_ref(); + + if let Some(idx) = self.sync_data.find_file_index(path, &self.base_path) { + let slot = if idx < self.sync_data.base_count { + FileSlot::Base(idx) + } else { + FileSlot::Overflow(idx) + }; + + return self.handle_file_modify(path, slot); + } + + self.add_new_file(path) + } + + #[tracing::instrument(skip_all, fields(path = ?path), level = Level::DEBUG)] + fn handle_file_modify(&mut self, path: &Path, slot: FileSlot) -> Option<&FileItem> { + let overlay = self.sync_data.bigram_overlay.as_ref().map(Arc::clone); + let pos = slot.index(); + + // this is the only way to actually know if the file is on disk, we CAN NOT + // rely on the watcher to proive the latest state of the file, do the actual check + let metadata = match std::fs::metadata(path) { + Ok(m) => { + self.untombstone_file(pos); + + m + } + Err(_) => { + self.tombstone_file(pos); + return None; + } + }; + + let (_arena, file) = self.sync_data.get_file_mut(pos)?; + + let size = metadata.len(); + let modified_time = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()); + + file.update_metadata(&self.cache_budget, modified_time, Some(size)); + + // Re-classify binary status from current content (chunked, fixed + // buffer). Already-binary files are left alone. + if !file.is_binary() { + let mut chunk = [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE]; + file.detect_binary_per_byte(path, &mut chunk); + } + + // Indexable base-region files feed fresh content to the bigram overlay. + if matches!(slot, FileSlot::Base(_)) + && let Some(ref overlay) = overlay + { + let in_indexable = { + let guard = overlay.read(); + pos < guard.base_file_count() + }; + + if in_indexable && let Ok(content) = std::fs::read(path) { + overlay.write().modify_file(pos, &content); + } + } + + self.sync_data.files().get(pos) + } + + /// Adds a new file to picker, if the file can not be added returns `None` + /// which indicates that it's time to trigger a new sync + #[tracing::instrument(skip(self))] + pub fn add_new_file(&mut self, path: &Path) -> Option<&FileItem> { + // On Windows `pathdiff::diff_paths` is byte-wise, so a short-name + // input never shares a prefix with the canonicalized base_path and + // the resulting relative path becomes absolute. Canonicalize first. + #[cfg(windows)] + let canonical_buf: Option = if path.starts_with(&self.base_path) { + None + } else if let Ok(c) = crate::path_utils::canonicalize(path) { + Some(c) + } else { + tracing::error!(path = ?path.display(), "Failed to canonicalize file path to add"); + return None; + }; + + #[cfg(windows)] + let path_for_index: &Path = canonical_buf.as_deref().unwrap_or(path); + #[cfg(not(windows))] + let path_for_index: &Path = path; + + let (mut file_item, rel_path) = + FileItem::new(path_for_index.to_path_buf(), &self.base_path, None); + + // we have to perform manual classification for every new file this will be + // batched during the scan, this is the path when the file is ad-hoc added to the sync + file_item.detect_binary_per_byte( + path_for_index, + // inline chunk buf + &mut [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE], + ); + + let builder = self.sync_data.overflow_builder.get_or_insert_with(|| { + // we know that overflow would never create more files during the file + crate::simd_path::ChunkedPathStoreBuilder::new(MAX_OVERFLOW_FILES) + }); + + file_item.set_path(builder.add_file_immediate(&rel_path, file_item.path.filename_offset)); + file_item.set_overflow(true); + + if !self.sync_data.files.push(file_item) { + return None; + } + + self.sync_data.live_count += 1; + self.sync_data.files.last() + } + + fn tombstone_file(&mut self, index: usize) { + let file = &mut self.sync_data.files[index]; + if file.is_deleted() { + return; + } + + file.set_deleted(true); + file.invalidate_mmap(&self.cache_budget); + file.git_status = None; + + // Only base-region files participate in the bigram overlay + if index < self.sync_data.base_count + && let Some(ref overlay) = self.sync_data.bigram_overlay + { + overlay.write().delete_file(index); + } + + self.sync_data.live_count -= 1; + } + + fn untombstone_file(&mut self, index: usize) { + let file = &mut self.sync_data.files[index]; + if !file.is_deleted() { + return; + } + file.set_deleted(false); + + self.sync_data.live_count += 1; + } + + /// Marks file as deleted, make sure that if you call this yourself these changes can be reverted + /// by the internal mechanics if the file actually exists on the disk, use only if you know that + /// the file going to be disapperaed or if you do not have the watcher installed + pub fn remove_file_by_path(&mut self, path: impl AsRef) -> bool { + let path = path.as_ref(); + if let Some(index) = self.sync_data.find_file_index(path, &self.base_path) { + self.tombstone_file(index); + true + } else { + false + } + } + + // TODO make this O(n) + pub fn remove_all_files_in_dir(&mut self, dir: impl AsRef) -> usize { + let dir_path = dir.as_ref(); + let relative_dir = self + .to_relative_path(dir_path) + .map(|c| c.into_owned()) + .unwrap_or_default(); + + let dir_prefix = if relative_dir.is_empty() { + String::new() + } else { + // Stored relative paths are '/'-canonical on every platform. + format!("{relative_dir}/") + }; + + self.sync_data.tombstone_files_with_arena(|file, arena| { + file.relative_path_starts_with(arena, &dir_prefix) + }) + } + + /// Use this to prevent any substantial background threads from acquiring the locks + pub fn cancel(&self) { + self.signals.cancelled.store(true, Ordering::Release); + } + + /// Stop the background filesystem watcher. Non-blocking. + pub fn stop_background_monitor(&mut self) { + if let Some(mut watcher) = self.background_watcher.take() { + watcher.stop(); + } + } + + /// Quick way to check if scan is going without acquiring a lock for [Self::get_scan_progress] + pub fn is_scan_active(&self) -> bool { + self.signals.scanning.load(Ordering::Relaxed) + } + + pub fn is_post_scan_active(&self) -> bool { + self.signals + .post_scan_indexing_active + .load(Ordering::Acquire) + } + + /// Return a clone of the watcher-ready flag so callers can poll it without + /// holding a lock on the picker. + pub fn watcher_signal(&self) -> Arc { + Arc::clone(&self.signals.watcher_ready) + } + + /// Convert an absolute path to a relative path string (relative to base_path). + /// Returns None if the path doesn't start with base_path. + /// + /// On Windows the picker canonicalizes its base via `dunce`, so caller + /// paths that still carry 8.3 short names or a different casing would + /// fail a naive prefix check. Fall back to canonicalizing (or, when the + /// file was just deleted, canonicalizing its parent) before stripping. + fn to_relative_path<'a>(&self, path: &'a Path) -> Option> { + if let Ok(stripped) = path.strip_prefix(&self.base_path) + && let Some(s) = stripped.to_str() + { + // Callers compare against '/'-canonical stored paths. + return Some(crate::path_utils::to_canonical_slashes(s)); + } + + #[cfg(windows)] + { + let rel = canonical_relative_path(path, &self.base_path)?; + return Some(std::borrow::Cow::Owned(rel)); + } + + #[cfg(not(windows))] + None + } +} + +/// Resolve a possibly-short-name Windows path to the picker's canonical base. +/// Used by the Windows-only fallbacks in `to_relative_path` and +/// `find_file_index` so events still match tombstoned entries. +#[cfg(windows)] +fn canonical_relative_path(path: &Path, base: &Path) -> Option { + if let Ok(canonical) = crate::path_utils::canonicalize(path) + && let Ok(stripped) = canonical.strip_prefix(base) + && let Some(s) = stripped.to_str() + { + return Some(crate::path_utils::to_canonical_slashes(s).into_owned()); + } + + // Deleted files can't be canonicalized — canonicalize the parent and + // re-attach the filename. + let parent = path.parent()?; + let file_name = path.file_name()?; + let canonical_parent = crate::path_utils::canonicalize(parent).ok()?; + let stripped_parent = canonical_parent.strip_prefix(base).ok()?; + let mut rel = stripped_parent.to_path_buf(); + rel.push(file_name); + rel.to_str() + .map(|s| crate::path_utils::to_canonical_slashes(s).into_owned()) +} + +impl Drop for FilePicker { + fn drop(&mut self) { + // Cancel any in-flight ScanJob bound to this picker's signals so + // it cannot mutate the replacement picker after a swap. + self.signals.cancelled.store(true, Ordering::Release); + // Wake the git-status consumer so it exits; never joined (it takes + // the picker write lock, a blocking join here could deadlock). + self.git_status_worker.signal_shutdown(); + } +} + +#[derive(Debug, Clone, Copy)] +enum FileSlot { + Base(usize), + Overflow(usize), +} + +impl FileSlot { + fn index(self) -> usize { + match self { + FileSlot::Base(i) | FileSlot::Overflow(i) => i, + } + } +} + +/// Snapshot of FilePicker state for off-lock post-scan work. +/// +/// Each data field is an Arc-shared clone of the picker's backing +/// allocation, so dropping the `FilePicker` (e.g. via +/// `SharedFilePicker::write().take()`) cannot free memory this +/// snapshot is still reading — UAF is impossible by construction. +/// +/// Implements `Drop` to clear `post_scan_indexing_active`. Since only +/// one snapshot can exist at a time (enforced by the flag check in +/// `post_scan_snapshot`) and it is always created/dropped within +/// `ScanJob::run`, `scan_job_running == false` implies no live snapshot. +pub(crate) struct PostScanUnsafeSnapshot { + pub files: StableVec, + pub arena: Option>, + // TODO figure this out + pub _budget: Arc, + pub base_count: usize, + pub indexable_count: usize, + pub base_path: PathBuf, + post_scan_flag: Arc, +} + +impl Drop for PostScanUnsafeSnapshot { + fn drop(&mut self) { + self.post_scan_flag.store(false, Ordering::Release); + } +} + +// SAFETY: every data field is Arc-shared and outlives the snapshot +// via its own refcount. The mutable cast in `apply_git_status_and_frecency` +// is consumed on the scan thread under the single-writer discipline. +unsafe impl Send for PostScanUnsafeSnapshot {} +unsafe impl Sync for PostScanUnsafeSnapshot {} + +/// A point-in-time snapshot of the file-scanning progress. +/// +/// Returned by [`FilePicker::get_scan_progress`]. Useful for displaying +/// a progress indicator while the initial scan is running. +#[derive(Debug, Clone)] +pub struct ScanProgress { + pub scanned_files_count: usize, + pub is_scanning: bool, + pub is_watcher_ready: bool, + pub is_warmup_complete: bool, +} + +impl FileSync { + pub(crate) fn discover_git_workdir(base_path: &Path) -> Option { + let git_workdir = Repository::discover(base_path) + .ok() + .and_then(|repo| repo.workdir().map(Path::to_path_buf)) + .map(crate::path_utils::normalize); + + match &git_workdir { + Some(workdir) => debug!("Git repository found at: {}", workdir.display()), + None => warn!("No git repository found for path: {}", base_path.display()), + } + + git_workdir + } + + pub(crate) fn spawn_git_status(git_workdir: PathBuf) -> JoinHandle> { + std::thread::spawn(move || { + GitStatusCache::read_git_status( + Some(git_workdir.as_path()), + &mut crate::git::initial_scan_status_options(), + ) + }) + } + + /// Returns files immediately (searchable) and a handle to the in-progress + /// git status computation. This avoids blocking on `git status` which can + /// take 10+ seconds on very large repos (e.g. chromium). + #[tracing::instrument(skip_all, name = "walk_filesystem", level = Level::INFO)] + pub(crate) fn walk_filesystem( + base_path: &Path, + git_workdir: Option, + synced_files_count: &Arc, + shared_frecency: &SharedFrecency, + mode: FFFMode, + follow_symlinks: bool, + ) -> Result { + let scan_start = std::time::Instant::now(); + info!("SCAN: Starting filesystem walk and git status (async)"); + + // Walk files (the fast part, typically 2-3s even on huge repos). + let is_git_repo = git_workdir.is_some(); + let bg_threads = BACKGROUND_THREAD_POOL.current_num_threads(); + + let mut walk_output = crate::walk::walk_collect_files( + base_path, + is_git_repo, + follow_symlinks, + bg_threads, + synced_files_count, + )?; + let ignore_rules = walk_output.ignore_rules.take().map(Arc::new); + let mut pairs = walk_output.pairs; + + // Sort by (dir_part, filename). This groups files by their directory + // into contiguous runs so the linear dir-extraction pass below can + // dedupe by comparing only against the previous dir. + BACKGROUND_THREAD_POOL.install(|| { + pairs.par_sort_unstable_by(|(a, path_a), (b, path_b)| { + // SAFETY: `filename_offset` is always at a character boundary + let (a_dir, a_file) = path_a.split_at(a.path.filename_offset as usize); + let (b_dir, b_file) = path_b.split_at(b.path.filename_offset as usize); + a_dir.cmp(b_dir).then_with(|| a_file.cmp(b_file)) + }); + }); + + let mut builder = crate::simd_path::ChunkedPathStoreBuilder::new(pairs.len()); + let dirs = populates_dirs_files_chunked_storage(&mut pairs, &mut builder); + + let mut files: Vec = pairs.into_iter().map(|(file, _)| file).collect(); + let chunked_paths = builder.finish(); + let arena = chunked_paths.as_arena_ptr(); + + // Apply frecency scores (access-based only — git status not yet available). + // DirItem.max_access_frecency is AtomicI32, so parallel threads write directly. + let frecency = shared_frecency + .read() + .map_err(|_| Error::AcquireFrecencyLock)?; + + if let Some(frecency) = frecency.as_ref() { + let dirs_ref = &dirs; + BACKGROUND_THREAD_POOL.install(|| { + files.par_iter_mut().for_each(|file| { + let _ = file.update_frecency_scores(frecency, arena, base_path, mode); + let score = file.access_frecency_score as i32; + if score > 0 { + let dir_idx = file.parent_dir_index as usize; + if let Some(dir) = dirs_ref.get(dir_idx) { + dir.update_frecency_if_larger(score); + } + } + }); + }); + } + drop(frecency); + + // un-indexable files that are binary or not fitting the size cap has to beplaced in the end + let is_indexable = |f: &FileItem| { + !f.is_binary() + && f.size > 0 + && f.size <= crate::constants::MAX_INDEXABLE_FILE_SIZE as u64 + }; + + BACKGROUND_THREAD_POOL.install(|| { + files.par_sort_unstable_by(|a, b| { + (!is_indexable(a)) + .cmp(&!is_indexable(b)) + // this just makes it faster in terms of allocation - we store the dir indexes + .then_with(|| a.parent_dir_index.cmp(&b.parent_dir_index)) + .then_with(|| a.file_name(arena).cmp(&b.file_name(arena))) + }); + }); + let indexable_count = files.partition_point(is_indexable); + + // Ask the allocator to return freed pages to the OS. + hint_allocator_collect(); + + let file_item_size = std::mem::size_of::(); + let files_vec_bytes = files.len() * file_item_size; + let dir_table_bytes = dirs.len() * std::mem::size_of::() + + dirs + .iter() + .map(|d| d.relative_path(arena).len()) + .sum::(); + + let total_time = scan_start.elapsed(); + info!( + "SCAN: Walk completed in {:?} ({} files, {} dirs, \ + chunked_store={:.2}MB, files_vec={:.2}MB, dirs={:.2}MB, FileItem={}B)", + total_time, + files.len(), + dirs.len(), + chunked_paths.heap_bytes() as f64 / 1_048_576.0, + files_vec_bytes as f64 / 1_048_576.0, + dir_table_bytes as f64 / 1_048_576.0, + file_item_size, + ); + + let base_count = files.len(); + + Ok(FileSync { + files: StableVec::from_vec_with_reserve(files, MAX_OVERFLOW_FILES), + indexable_count, + base_count, + live_count: base_count, + dirs: StableVec::from_vec_with_reserve(dirs, 0), + overflow_builder: None, + git_workdir, + bigram_index: None, + bigram_overlay: None, + chunked_paths: Some(Arc::new(chunked_paths)), + ignore_rules, + }) + } +} + +/// Pre-populate mmap caches for cold tail files so the first grep search +/// doesn't pay the mmap creation + page fault cost. +#[allow(dead_code)] +#[tracing::instrument(skip(files), name = "warmup_mmaps", level = Level::DEBUG)] +pub(crate) fn warmup_mmaps( + files: &[FileItem], + budget: &ContentCacheBudget, + base_path: &Path, + arena: ArenaPtr, +) { + // for most of the use cases mmaps limit would be significantly smaller than arepo + for file in files.iter() { + if file.is_likely_hot() + || file.is_binary() + || file.size == 0 + || file.size > budget.max_file_size + { + continue; + } + + let _ = file.get_cached_content(arena, base_path, budget); + + if budget.is_exhausted() { + break; + } + } +} + +/// This does both thing (yes sorry all the OOP morons) +/// in one go: populates files chunked storage and creates new directories +fn populates_dirs_files_chunked_storage<'a>( + pairs: &'a mut [(FileItem, String)], + chunk_storage: &mut crate::simd_path::ChunkedPathStoreBuilder, +) -> Vec { + let mut dirs: Vec = Vec::new(); + + let mut prev_dir: &'a str = ""; + let mut prev_dir_valid = false; + let mut current_dir_idx: u32 = 0; + + for (file, rel) in pairs.iter_mut() { + let rel: &'a str = rel; + let dir_part: &'a str = &rel[..file.path.filename_offset as usize]; + + if !prev_dir_valid || prev_dir != dir_part { + let dir_string = chunk_storage.add_dir_immediate(dir_part); + + // Compute last-segment offset: for "src/components/" -> 4 (points to "components/") + let last_seg = if dir_part.is_empty() { + 0 + } else { + let trimmed = dir_part.trim_end_matches(std::path::is_separator); + trimmed + .rfind(std::path::is_separator) + .map(|i| i + 1) + .unwrap_or(0) as u16 + }; + + dirs.push(DirItem::new(dir_string, last_seg)); + current_dir_idx = (dirs.len() - 1) as u32; + + prev_dir = dir_part; + prev_dir_valid = true; + } + + file.path = chunk_storage.add_file_immediate(rel, file.path.filename_offset); + file.parent_dir_index = current_dir_idx; + } + + dirs +} + +/// Fast extension-based binary detection. Avoids opening files during scan. +/// Covers the vast majority of binary files in typical repositories. +#[inline] +#[doc(hidden)] +pub fn is_known_binary_extension(path: &Path) -> bool { + let Some(ext) = path.extension().and_then(|e| e.to_str()) else { + return false; + }; + is_binary_extension_str(ext) +} + +/// Like [`is_known_binary_extension`] but takes a basename string directly, +/// avoiding `Path::extension()` overhead. Mirrors `Path::extension()` +/// semantics: dotfiles with no other dots → no extension. Used by the zlob +/// walker, which already has the basename slice from traversal. +#[cfg(feature = "zlob")] +#[inline] +pub(crate) fn is_known_binary_extension_basename(name: &str) -> bool { + match name.rfind('.') { + Some(pos) if pos > 0 && pos < name.len() - 1 => is_binary_extension_str(&name[pos + 1..]), + _ => false, + } +} + +#[inline] +fn is_binary_extension_str(ext: &str) -> bool { + matches!( + ext, + // Images + "png" | "jpg" | "jpeg" | "gif" | "bmp" | "ico" | "webp" | "tiff" | "tif" | "avif" | + "heic" | "heif" | "jxl" | "jp2" | "j2k" | "psd" | "icns" | "cur" | "cr2" | + "nef" | "dng" | "tga" | + // GPU / VFX texture formats + "rgbe" | "hdr" | "exr" | "dds" | "ktx" | "ktx2" | "pvr" | "astc" | + // Adobe Illustrator (PDF wrapper) / Apple webarchive / MIME HTML archive + "ai" | "webarchive" | "mhtml" | + // Video/Audio + "mp4" | "avi" | "mov" | "wmv" | "mkv" | "mp3" | "wav" | "flac" | "ogg" | "m4a" | + "aac" | "webm" | "flv" | "mpg" | "mpeg" | "wma" | "opus" | "pcm" | "reapeaks" | + // Compressed/Archives + "zip" | "tar" | "gz" | "bz2" | "xz" | "7z" | "rar" | "zst" | "lz4" | "lzma" | + "cab" | "cpio" | "jsonlz4" | + // Packages/Installers + "deb" | "rpm" | "apk" | "dmg" | "msi" | "iso" | "nupkg" | "whl" | "egg" | + "appimage" | "flatpak" | "crx" | "pak" | + // Executables/Libraries + "exe" | "dll" | "so" | "dylib" | "o" | "a" | "lib" | "bin" | "elf" | + // Documents (binary office formats) + "pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" | + // Databases + "db" | "sqlite" | "sqlite3" | "mdb" | + // SQLite / LevelDB auxiliary files + "sqlite-wal" | "sqlite-shm" | "sqlite3-wal" | "sqlite3-shm" | + "db-wal" | "db-shm" | "ldb" | + // Fonts + "ttf" | "otf" | "woff" | "woff2" | "eot" | + // Compiled/Runtime + "class" | "pyc" | "pyo" | "wasm" | "dex" | "jar" | "war" | + // OCaml / Swift / Objective-C build artefacts + "cmi" | "cmt" | "cmti" | "cmx" | "nib" | + "swiftdeps" | "swiftdeps~" | "swiftdoc" | "swiftmodule" | "swiftsourceinfo" | + // ML/Data Science + "npy" | "npz" | "h5" | "hdf5" | "pt" | "onnx" | + "safetensors" | "tfrecord" | "tflite" | "gguf" | "ggml" | "joblib" | + // 3D/Game assets + "glb" | "blend" | "blp" | + // Gzipped-XML / binary maps + "dia" | "bcmap" | + // Protobuf wire format + "pb" | + // Data/serialized + "parquet" | "arrow" | + // IDE/OS metadata + "suo" + ) +} + +/// Length of the longest shared directory prefix of two relative dir +/// paths (without a trailing separator), measured as the number of bytes +/// up to and including the last shared separator — plus the full shorter +/// path when it is itself a directory prefix of the longer one. +/// +/// Examples: +/// `"src/components"` vs `"src/routes"` → 4 (`"src/"` emitted once) +/// `"lib/deep/nested"` vs `"lib/deep"` → 8 (`"lib/deep"` is a prefix) +/// `"lib/deep"` vs `"lib/deeper"` → 4 (only `"lib/"` is shared) +/// `"lib"` vs `"src"` → 0 +/// +/// Used by [`FilePicker::for_each_watch_dir`] to avoid re-emitting +/// ancestors that were already yielded for the previous (sorted) sibling. +fn common_dir_prefix_len(a: &str, b: &str) -> usize { + let max = a.len().min(b.len()); + let a_bytes = a.as_bytes(); + let b_bytes = b.as_bytes(); + let mut last_sep = 0; + let mut i = 0; + while i < max && a_bytes[i] == b_bytes[i] { + if std::path::is_separator(a_bytes[i] as char) { + last_sep = i + 1; + } + i += 1; + } + // If one string is a prefix of the other and the next byte in the + // longer one is a separator, the full shorter path is a shared dir. + if i == max && i > 0 { + let longer = if a.len() > b.len() { a_bytes } else { b_bytes }; + if i < longer.len() && std::path::is_separator(longer[i] as char) { + return i; + } + } + last_sep +} + +/// Ask the global allocator to return freed pages to the OS. +/// Enabled via the `mimalloc-collect` feature (set by fff-nvim). +/// No-op when the feature is off (tests, system allocator). +pub(crate) fn hint_allocator_collect() { + #[cfg(feature = "mimalloc-collect")] + { + // Collect BACKGROUND_THREAD_POOL workers — that's where the bigram + // builder allocated memory. `rayon::broadcast` would target the global + // pool, which is the wrong set of threads. + BACKGROUND_THREAD_POOL.broadcast(|_| unsafe { libmimalloc_sys::mi_collect(true) }); + + // Main thread too. + unsafe { libmimalloc_sys::mi_collect(true) }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The watcher must watch every ancestor directory up to `base_path`, + /// not just the immediate parents of indexed files. Intermediate dirs + /// that contain only subdirectories (no direct files) are NOT in + /// `sync_data.dirs` — yet they must still appear in `extract_watch_dirs` + /// so Create events on new subdirectories below them fire. + /// + /// Correctness regression guard for any refactor that replaces the + /// ancestor walk with a direct `sync_data.dirs` iteration. + #[test] + fn extract_watch_dirs_includes_pure_ancestor_dirs() { + let dir = tempfile::tempdir().unwrap(); + // On Windows the picker canonicalizes base_path with dunce; match that + // here so the stored dir paths line up with assertions built from + // `base.join(..)` (which otherwise would carry an 8.3 short name). + let base_buf = crate::path_utils::canonicalize(dir.path()).unwrap(); + let base = base_buf.as_path(); + + // Tree: + // base/src/components/button.txt (src/components has a file) + // base/src/routes/home.txt (src/routes has a file) + // base/lib/deep/nested/util.txt (lib and lib/deep have no files) + // + // `sync_data.dirs` will only contain: + // src/components/ + // src/routes/ + // lib/deep/nested/ + // + // But the watcher also needs: + // src/ (pure ancestor — no direct files) + // lib/ (pure ancestor) + // lib/deep/ (pure ancestor) + // otherwise new siblings like `src/NewDir/x.txt` are missed. + for rel in [ + "src/components/button.txt", + "src/routes/home.txt", + "lib/deep/nested/util.txt", + ] { + let path = base.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"x").unwrap(); + } + + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_str().unwrap().into(), + watch: false, + ..Default::default() + }) + .unwrap(); + picker.collect_files().unwrap(); + + let mut watch_dirs: Vec = Vec::new(); + picker.for_each_dir(|p| { + watch_dirs.push(p.to_path_buf()); + std::ops::ControlFlow::Continue(()) + }); + let watch_set: std::collections::HashSet = watch_dirs.iter().cloned().collect(); + + // Immediate parents (in sync_data.dirs) must be present. + for rel in ["src/components", "src/routes", "lib/deep/nested"] { + assert!( + watch_set.contains(&base.join(rel)), + "expected immediate parent {rel} in watch dirs, got {watch_set:?}", + ); + } + + // Pure-ancestor dirs (NOT in sync_data.dirs) must also be present. + for rel in ["src", "lib", "lib/deep"] { + assert!( + watch_set.contains(&base.join(rel)), + "expected pure-ancestor {rel} in watch dirs, got {watch_set:?}", + ); + } + + // No duplicates — streaming dedup must not emit the same dir twice. + assert_eq!( + watch_dirs.len(), + watch_set.len(), + "duplicate watch dir emitted: {watch_dirs:?}", + ); + + // Base path itself is NOT walked into the result — the walker stops + // at `current == base`. The outer `debouncer.watch(base_path, ...)` + // call in create_debouncer covers it separately. + assert!( + !watch_set.contains(base), + "base path must not be in watch dirs (covered by the top-level watch call)", + ); + } + + #[test] + fn common_dir_prefix_len_cases() { + assert_eq!(common_dir_prefix_len("", ""), 0); + assert_eq!(common_dir_prefix_len("", "src"), 0); + assert_eq!(common_dir_prefix_len("lib", "src"), 0); + assert_eq!(common_dir_prefix_len("src/components", "src/routes"), 4); + assert_eq!(common_dir_prefix_len("lib/deep/nested", "lib/deep"), 8); + assert_eq!(common_dir_prefix_len("lib/deep", "lib/deep/nested"), 8); + assert_eq!(common_dir_prefix_len("lib/deep", "lib/deeper"), 4); + assert_eq!(common_dir_prefix_len("src", "src"), 0); + // "src" is emitted-as-dir; "src/x" extends it — full "src" is shared. + assert_eq!(common_dir_prefix_len("src", "src/x"), 3); + } +} diff --git a/crates/fff-core/src/git.rs b/crates/fff-core/src/git.rs new file mode 100644 index 0000000..85bce5e --- /dev/null +++ b/crates/fff-core/src/git.rs @@ -0,0 +1,253 @@ +use crate::error::Result; +use ahash::AHashMap; +use git2::{Repository, Status, StatusOptions}; +use std::{ + fmt::Debug, + path::{Path, PathBuf}, +}; + +pub(crate) fn default_status_options() -> StatusOptions { + let mut opts = StatusOptions::new(); + opts.include_untracked(true) + .recurse_untracked_dirs(true) + .include_unmodified(true) + .exclude_submodules(true); + opts +} + +/// Status options for the initial scan / rescan. +/// +/// Skips `include_unmodified` because every `FileItem` starts with +/// `git_status: None` (== clean), so a missing cache entry already means +/// "clean" — no need to ask libgit2 to enumerate every tracked path. +/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries). +pub(crate) fn initial_scan_status_options() -> StatusOptions { + let mut opts = StatusOptions::new(); + opts.include_untracked(true) + .recurse_untracked_dirs(true) + .exclude_submodules(true); + opts +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct GitStatusCache(AHashMap); + +impl IntoIterator for GitStatusCache { + type Item = (PathBuf, Status); + type IntoIter = as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl GitStatusCache { + pub fn statuses_len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn lookup_status(&self, full_path: &Path) -> Option { + self.0.get(full_path).copied() + } + + #[tracing::instrument(skip(repo, status_options))] + fn read_status_impl(repo: &Repository, status_options: &mut StatusOptions) -> Result { + let statuses = repo.statuses(Some(status_options))?; + let Some(repo_path) = repo.workdir() else { + return Ok(Self(AHashMap::new())); // repo is bare + }; + + let repo_path = crate::path_utils::normalize(repo_path.to_path_buf()); + + let mut entries = AHashMap::with_capacity(statuses.len()); + for entry in &statuses { + if let Some(entry_path) = entry.path() { + // libgit2 returns entry paths with forward slashes on every platform + // fff stores native paths - meaning we have forward slash issue on windows + let full_path = crate::path_utils::normalize(repo_path.join(entry_path)); + entries.insert(full_path, entry.status()); + } + } + + Ok(Self(entries)) + } + + pub fn read_git_status( + git_workdir: Option<&Path>, + status_options: &mut StatusOptions, + ) -> Option { + let git_workdir = git_workdir.as_ref()?; + let repository = Repository::open(git_workdir).ok()?; + + let status = Self::read_status_impl(&repository, status_options); + + match status { + Ok(status) => Some(status), + Err(e) => { + tracing::error!(?e, "Failed to read git status"); + + None + } + } + } + + #[tracing::instrument(skip(repo), fields(paths_count = paths.len()), level = tracing::Level::DEBUG)] + pub fn git_status_for_paths + Debug>( + repo: &Repository, + paths: &[TPath], + ) -> Result { + if paths.is_empty() { + return Ok(Self(AHashMap::new())); + } + + let Some(workdir) = repo.workdir() else { + return Ok(Self(AHashMap::new())); + }; + let workdir = crate::path_utils::normalize(workdir.to_path_buf()); + + // git pathspec is pretty slow and requires to walk the whole directory + // so for a single file which is the most general use case we query directly the file + if paths.len() == 1 { + let full_path = paths[0].as_ref(); + let relative_path = full_path.strip_prefix(&workdir)?; + let status = repo.status_file(relative_path)?; + + let mut map = AHashMap::with_capacity(1); + map.insert(full_path.to_path_buf(), status); + return Ok(Self(map)); + } + + let mut status_options = default_status_options(); + for path in paths { + status_options.pathspec(path.as_ref().strip_prefix(&workdir)?); + } + + let git_status_cache = Self::read_status_impl(repo, &mut status_options)?; + Ok(git_status_cache) + } +} + +#[inline] +pub fn is_modified_status(status: Status) -> bool { + status.intersects( + Status::WT_MODIFIED + | Status::INDEX_MODIFIED + | Status::WT_NEW + | Status::INDEX_NEW + | Status::WT_RENAMED, + ) +} + +pub fn format_git_status_opt(status: Option) -> Option<&'static str> { + match status { + None => Some("clean"), + Some(status) => { + if status.contains(Status::WT_NEW) { + Some("untracked") + } else if status.contains(Status::WT_MODIFIED) { + Some("modified") + } else if status.contains(Status::WT_DELETED) { + Some("deleted") + } else if status.contains(Status::WT_RENAMED) { + Some("renamed") + } else if status.contains(Status::INDEX_NEW) { + Some("staged_new") + } else if status.contains(Status::INDEX_MODIFIED) { + Some("staged_modified") + } else if status.contains(Status::INDEX_DELETED) { + Some("staged_deleted") + } else if status.contains(Status::IGNORED) { + Some("ignored") + } else if status.contains(Status::CURRENT) || status.is_empty() { + Some("clean") + } else { + None + } + } + } +} + +pub fn format_git_status(status: Option) -> &'static str { + format_git_status_opt(status).unwrap_or("unknown") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::process::Command; + use tempfile::TempDir; + + fn git(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?} failed"); + } + + /// Regression: on case-insensitive filesystems libgit2 returns + /// statuses in a case-insensitive order. Our previous sorted-`Vec` + + /// `binary_search_by(Path::cmp)` lookup silently missed entries + /// because `Path::cmp` is byte-wise. + /// + /// This test uses deliberately mixed-case filenames so the two + /// orderings disagree, then checks every lookup succeeds. + #[test] + fn lookup_is_case_exact_regardless_of_libgit2_sort_order() { + let tmp = TempDir::new().unwrap(); + // `std::fs::canonicalize` on Windows adds a `\\?\` UNC prefix that + // libgit2's workdir string lacks. Use dunce so both sides match. + let base = crate::path_utils::canonicalize(tmp.path()).unwrap(); + + // Mixed-case names that sort differently under byte-wise vs + // case-insensitive comparators. + let names = [ + "README.md", + "a_lower.rs", + "Z_upper.rs", + "mixed_Case.txt", + "nested/Inner_File.rs", + ]; + for n in &names { + let p = base.join(n); + fs::create_dir_all(p.parent().unwrap()).unwrap(); + fs::write(&p, format!("// {n}\n")).unwrap(); + } + + git(&base, &["init", "-b", "main"]); + git(&base, &["add", "-A"]); + git(&base, &["commit", "-m", "seed", "--no-gpg-sign"]); + + // Modify every file so they all end up in the status output as + // WT_MODIFIED — guarantees a non-trivial map we have to look up. + for n in &names { + let p = base.join(n); + fs::write(&p, format!("// {n}\n// edit\n")).unwrap(); + } + + let repo = Repository::open(&base).unwrap(); + let paths: Vec = names.iter().map(|n| base.join(n)).collect(); + let cache = GitStatusCache::git_status_for_paths(&repo, &paths).unwrap(); + + for (n, abs) in names.iter().zip(paths.iter()) { + let status = cache.lookup_status(abs); + assert!( + status.is_some(), + "lookup for {n} returned None; cache holds {} entries", + cache.statuses_len(), + ); + assert!( + status.unwrap().contains(Status::WT_MODIFIED), + "expected WT_MODIFIED for {n}, got {:?}", + status + ); + } + } +} diff --git a/crates/fff-core/src/git_status_worker.rs b/crates/fff-core/src/git_status_worker.rs new file mode 100644 index 0000000..b6e7402 --- /dev/null +++ b/crates/fff-core/src/git_status_worker.rs @@ -0,0 +1,120 @@ +use crate::shared::{SharedFrecency, WeakFilePicker}; +use ahash::AHashSet; +use parking_lot::{Condvar, Mutex}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +// we don't really need a queue here +#[derive(Default)] +struct Pending { + paths: AHashSet, + full_rescan: bool, + shutdown: bool, +} + +impl Pending { + fn has_work(&self) -> bool { + self.full_rescan || !self.paths.is_empty() + } +} + +/// Condvar based queue that is used for batch processing events +pub(crate) struct GitStatusWorker { + state: Mutex, + cv: Condvar, + consumer_spawned: AtomicBool, +} + +impl GitStatusWorker { + pub(crate) fn new() -> Arc { + Arc::new(Self { + state: Mutex::new(Pending::default()), + cv: Condvar::new(), + consumer_spawned: AtomicBool::new(false), + }) + } + + pub(crate) fn spawn_once( + self: &Arc, + weak_picker: WeakFilePicker, + frecency: SharedFrecency, + ) { + if self + .consumer_spawned + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + Self::spawn_consumer(Arc::clone(self), weak_picker, frecency); + } + } + + pub(crate) fn enqueue_paths(&self, paths: I) + where + I: IntoIterator, + { + let mut guard = self.state.lock(); + guard.paths.extend(paths); + drop(guard); + self.cv.notify_one(); + } + + pub(crate) fn request_full_rescan(&self) { + let mut guard = self.state.lock(); + guard.full_rescan = true; + drop(guard); + self.cv.notify_one(); + } + + pub(crate) fn signal_shutdown(&self) { + let mut guard = self.state.lock(); + guard.shutdown = true; + drop(guard); + self.cv.notify_one(); + } + + fn wait_and_take(&self) -> Option { + let mut guard = self.state.lock(); + while !guard.shutdown && !guard.has_work() { + self.cv.wait(&mut guard); + } + if guard.shutdown { + return None; + } + Some(std::mem::take(&mut *guard)) + } + + // the problem: git status update can take a lot of time especially on big repositories + // and there is unpredictable wait time on the lock file if huge commit is going so we have to + // spawn a separate thread to guartee that notify handler is unlocked even if git update takes a + // lot of time on every event burst (pretty cheap as this thread is going to sleep 99.9% of time) + fn spawn_consumer( + mailbox: Arc, + weak_picker: WeakFilePicker, + frecency: SharedFrecency, + ) { + let _ = std::thread::Builder::new() + .name("fff-git-status".into()) + .spawn(move || { + while let Some(work) = mailbox.wait_and_take() { + let Some(picker) = weak_picker.upgrade() else { + break; + }; + + if work.full_rescan { + if let Err(e) = picker.refresh_git_status(&frecency) { + tracing::error!("git-status worker: full rescan failed: {e:?}"); + } + } else if !work.paths.is_empty() { + let paths: Vec = work.paths.into_iter().collect(); + if let Err(e) = picker.update_git_status_for_paths(&paths, &frecency) { + tracing::error!("git-status worker: path update failed: {e:?}"); + } + } + } + + tracing::info!("git-status worker stopped"); + }) + .inspect_err(|err| tracing::error!(?err, "Failed to spawn git status worker")); + } +} diff --git a/crates/fff-core/src/grep/classify.rs b/crates/fff-core/src/grep/classify.rs new file mode 100644 index 0000000..8619a03 --- /dev/null +++ b/crates/fff-core/src/grep/classify.rs @@ -0,0 +1,131 @@ +//! Definition and import line classification (vibe coded POC) +//! +//! Byte-level heuristics that tag a matched line as a code definition +//! (`struct`, `fn`, `class`, …) or an import/use statement. Used to +//! rank/annotate grep results for AI/MCP consumers. Gated behind the +//! `definitions` feature since only such consumers need it. + +/// Detect if a line looks like a code definition (struct, fn, class, etc.) +pub fn is_definition_line(line: &str) -> bool { + let s = line.trim_start().as_bytes(); + let s = skip_modifiers(s); + is_definition_keyword(s) +} + +/// Modifier keywords that can precede a definition keyword. +/// Each must be followed by whitespace to be consumed. +const MODIFIERS: &[&[u8]] = &[ + b"pub", + b"export", + b"default", + b"async", + b"abstract", + b"unsafe", + b"static", + b"protected", + b"private", + b"public", +]; + +/// Definition keywords to detect. +const DEF_KEYWORDS: &[&[u8]] = &[ + b"struct", + b"fn", + b"enum", + b"trait", + b"impl", + b"class", + b"interface", + b"function", + b"def", + b"func", + b"type", + b"module", + b"object", +]; + +/// Skip zero or more modifier keywords (including `pub(crate)` style visibility). +fn skip_modifiers(mut s: &[u8]) -> &[u8] { + loop { + // Handle `pub(...)` — e.g. `pub(crate)`, `pub(super)` + if s.starts_with(b"pub(") + && let Some(end) = s.iter().position(|&b| b == b')') + { + s = skip_ws(&s[end + 1..]); + continue; + } + let mut matched = false; + for &kw in MODIFIERS { + if s.starts_with(kw) { + let rest = &s[kw.len()..]; + if rest.first().is_some_and(|b| b.is_ascii_whitespace()) { + s = skip_ws(rest); + matched = true; + break; + } + } + } + if !matched { + return s; + } + } +} + +/// Check if `s` starts with a definition keyword followed by a word boundary. +fn is_definition_keyword(s: &[u8]) -> bool { + for &kw in DEF_KEYWORDS { + if s.starts_with(kw) { + let after = s.get(kw.len()); + // Word boundary: end of input, or next byte is not alphanumeric/underscore + if after.is_none_or(|b| !b.is_ascii_alphanumeric() && *b != b'_') { + return true; + } + } + } + false +} + +/// Skip ASCII whitespace. +#[inline] +fn skip_ws(s: &[u8]) -> &[u8] { + let n = s + .iter() + .position(|b| !b.is_ascii_whitespace()) + .unwrap_or(s.len()); + &s[n..] +} + +/// Detect import/use lines — lower value than definitions or usages. +/// +/// Checks if the line (after leading whitespace) starts with a common +/// import statement prefix. Pure byte-level checks, no regex. +pub fn is_import_line(line: &str) -> bool { + let s = line.trim_start().as_bytes(); + s.starts_with(b"import ") + || s.starts_with(b"import\t") + || (s.starts_with(b"from ") && s.get(5).is_some_and(|&b| b == b'\'' || b == b'"')) + || s.starts_with(b"use ") + || s.starts_with(b"use\t") + || starts_with_require(s) + || starts_with_include(s) +} + +/// Match `require(` or `require (`. +#[inline] +fn starts_with_require(s: &[u8]) -> bool { + if !s.starts_with(b"require") { + return false; + } + let rest = &s[b"require".len()..]; + rest.first() == Some(&b'(') || (rest.first() == Some(&b' ') && rest.get(1) == Some(&b'(')) +} + +/// Match `# include ` (with optional spaces after `#`). +#[inline] +fn starts_with_include(s: &[u8]) -> bool { + if s.first() != Some(&b'#') { + return false; + } + let rest = skip_ws(&s[1..]); + rest.starts_with(b"include ") || rest.starts_with(b"include\t") +} diff --git a/crates/fff-core/src/grep/fuzzy_grep.rs b/crates/fff-core/src/grep/fuzzy_grep.rs new file mode 100644 index 0000000..75e1f8f --- /dev/null +++ b/crates/fff-core/src/grep/fuzzy_grep.rs @@ -0,0 +1,358 @@ +use crate::simd_path::ArenaPtr; +use crate::types::{ContentCacheBudget, FileItem, MmapSlot}; +use fff_grep::lines::LineStep; +use rayon::prelude::*; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; + +use super::grep::{ + GrepMatch, GrepSearchOptions, char_indices_to_byte_offsets, classify_definition, + truncate_display_bytes, +}; +use super::utils::{GrepResult, strip_line_terminators}; + +#[allow(clippy::too_many_arguments)] +pub(super) fn fuzzy_grep_search<'a>( + grep_text: &str, + files_to_search: &[&'a FileItem], + options: &GrepSearchOptions, + total_files: usize, + filtered_file_count: usize, + case_insensitive: bool, + budget: &ContentCacheBudget, + abort_signal: &AtomicBool, + base_path: &Path, + arena: ArenaPtr, + overflow_arena: ArenaPtr, +) -> GrepResult<'a> { + // max_typos controls how many *needle* characters can be unmatched. + // A transposition (e.g. "shcema" -> "schema") costs ~1 typo with + // default gap penalties. We scale max_typos by needle length: + // 1-2 chars -> 0 typos (exact subsequence only) + // 3-5 chars -> 1 typo + // 6+ chars -> 2 typos + // Cap at 2: higher values (3+) let the SIMD prefilter pass lines + // missing key characters entirely (e.g. query "flvencodeX" matching + // lines without 'l' or 'v'). Quality comes from the post-match filters. + let max_typos = (grep_text.len() / 3).min(2); + let scoring = neo_frizbee::Scoring { + // Use default gap penalties. Higher values (e.g. 20) cause + // smith-waterman to prefer *dropping needle chars* over paying + // gap costs, which inflates the typo count and breaks + // transposition matching ("shcema" -> "schema" becomes 3 typos instead of 1) + exact_match_bonus: 100, + // gap_open_penalty: 4, + // gap_extend_penalty: 2, + prefix_bonus: 0, + capitalization_bonus: if case_insensitive { 0 } else { 4 }, + ..neo_frizbee::Scoring::default() + }; + + let matcher = neo_frizbee::Matcher::new( + grep_text, + &neo_frizbee::Config { + // Use the real max_typos so frizbee's SIMD prefilter actually rejects non-matching lines (~2 SIMD instructions per line vs full SW scoring). + max_typos: Some(max_typos as u16), + sort: false, + scoring, + ..Default::default() + }, + ); + + // Minimum score threshold: 50% of a perfect contiguous match. + // With default scoring (match_score=12, matching_case_bonus=4 = 16/char), + // a transposition costs ~5 from a gap, keeping the score well above 50% + let perfect_score = (grep_text.len() as u16) * 16; + let min_score = (perfect_score * 50) / 100; + + // Target identifiers are often longer than the query due to delimiters + // (e.g. query "flvencodepicture" -> "ff_flv_encode_picture_header" from ffmpeg) + // Allow 3x needle length to accommodate underscore/dot-separated names + let max_match_span = grep_text.len() * 3; + let needle_len = grep_text.len(); + + // Each delimiter (_, .) in the target creates a gap. A typical C/Rust + // identifier like "ff_flv_encode_picture_header" has 4-5 underscores. + // Scale generously so delimiter gaps don't reject valid matches. + let max_gaps = (needle_len / 3).max(2); + + // If a file doesn't contain enough distinct needle characters just skip it + let needle_bytes = grep_text.as_bytes(); + let mut unique_needle_chars: Vec = Vec::new(); + for &b in needle_bytes { + let lo = b.to_ascii_lowercase(); + let hi = b.to_ascii_uppercase(); + if !unique_needle_chars.contains(&lo) { + unique_needle_chars.push(lo); + } + if lo != hi && !unique_needle_chars.contains(&hi) { + unique_needle_chars.push(hi); + } + } + + // How many distinct needle chars must appear in the file. + // With max_typos allowed, we need at least (unique_count - max_typos) + let unique_count = { + let mut seen = [false; 256]; + for &b in needle_bytes { + seen[b.to_ascii_lowercase() as usize] = true; + } + seen.iter().filter(|&&v| v).count() + }; + let min_chars_required = unique_count.saturating_sub(max_typos); + + let time_budget = if options.time_budget_ms > 0 { + Some(std::time::Duration::from_millis(options.time_budget_ms)) + } else { + None + }; + let search_start = std::time::Instant::now(); + let budget_exceeded = AtomicBool::new(false); + let max_matches_per_file = options.max_matches_per_file; + + // for fuzzy match we need a bit smarter chunking as the amount of work we have to perform is + // exponentially larger than the original grep (and the nature of work is heavier), so in short we have to + // understand if the approximate index prefilter got us a lot of candidates or not + // + // if we have a few candidates -> likely we have a lot of matches, so verify the check faster + // if we have a lot of candidates -> rely on a larger chunk pipelining more parallel lines at once + let page_limit = options.page_limit; + let base_chunk = rayon::current_num_threads() * 4; + let prefilter_strong = total_files > 0 && files_to_search.len() * 2 < total_files; + let max_chunk = if prefilter_strong { + base_chunk + } else { + (base_chunk * 256).max(8 * 1024) + }; + + let growth = if prefilter_strong { 1 } else { 2 }; + let mut chunk_size = base_chunk; + let mut chunk_start = 0; + let mut running_matches = 0usize; + let mut per_file_results: Vec<(usize, &'a FileItem, Vec)> = Vec::new(); + + while chunk_start < files_to_search.len() { + let chunk_end = (chunk_start + chunk_size).min(files_to_search.len()); + let chunk = &files_to_search[chunk_start..chunk_end]; + let chunk_offset = chunk_start; + chunk_start = chunk_end; + chunk_size = (chunk_size * growth).min(max_chunk); + + // Parallel phase with `map_init`: each rayon worker thread clones the + // matcher once and gets a reusable read buffer + mmap slot. Buffer holds + // small files, slot holds fresh mmap for cache-miss files ≥ FRESH_MMAP_THRESHOLD. + let chunk_results: Vec<(usize, &'a FileItem, Vec)> = chunk + .par_iter() + .enumerate() + .map_init( + || { + ( + matcher.clone(), + Vec::with_capacity(64 * 1024), + MmapSlot::default(), + ) + }, + |(matcher, buf, mmap_slot), (local_idx, file)| { + if abort_signal.load(Ordering::Relaxed) { + budget_exceeded.store(true, Ordering::Relaxed); + return None; + } + + if let Some(budget) = time_budget + && search_start.elapsed() > budget + { + budget_exceeded.store(true, Ordering::Relaxed); + return None; + } + + let file_arena = if file.is_overflow() { + overflow_arena + } else { + arena + }; + + let file_bytes = + file.get_content_for_search(buf, mmap_slot, file_arena, base_path, budget)?; + + if min_chars_required > 0 { + let mut chars_found = 0usize; + for &ch in &unique_needle_chars { + if memchr::memchr(ch, file_bytes).is_some() { + chars_found += 1; + if chars_found >= min_chars_required { + break; + } + } + } + if chars_found < min_chars_required { + return None; + } + } + + // Validate the whole file as UTF-8 once upfront. Source code + // files are virtually always valid UTF-8; this single check + // replaces per-line from_utf8 calls (~8% of fuzzy grep time) + let file_is_utf8 = std::str::from_utf8(file_bytes).is_ok(); + + let mut stepper = LineStep::new(b'\n', 0, file_bytes.len()); + let estimated_lines = (file_bytes.len() / 40).max(64); + let mut file_lines: Vec<&str> = Vec::with_capacity(estimated_lines); + let mut line_meta: Vec<(u64, u64)> = Vec::with_capacity(estimated_lines); + + let mut line_number: u64 = 1; + while let Some(line_match) = stepper.next_match(file_bytes) { + let byte_offset = line_match.start() as u64; + let trimmed = strip_line_terminators(&file_bytes[line_match]); + + if !trimmed.is_empty() { + // we know for sure that the file is UTF-8 at this point + let line_str = if file_is_utf8 { + unsafe { std::str::from_utf8_unchecked(trimmed) } + } else if let Ok(s) = std::str::from_utf8(trimmed) { + s + } else { + line_number += 1; + continue; + }; + file_lines.push(line_str); + line_meta.push((line_number, byte_offset)); + } + + line_number += 1; + } + + if file_lines.is_empty() { + return None; + } + + // Single-pass: score + indices in one Smith-Waterman run per line (not parallel) + let matches_with_indices = matcher.match_list_indices(&file_lines); + let mut file_matches: Vec = Vec::new(); + + for mut match_indices in matches_with_indices { + if match_indices.score < min_score { + continue; + } + + let idx = match_indices.index as usize; + let raw_line = file_lines[idx]; + + let truncated = truncate_display_bytes(raw_line.as_bytes()); + let display_line = if truncated.len() < raw_line.len() { + // SAFETY: truncate_display_bytes preserves UTF-8 char boundaries + &raw_line[..truncated.len()] + } else { + raw_line + }; + + // If the line was truncated, re-compute indices on the shorter string. + if display_line.len() < raw_line.len() { + let Some(re_indices) = matcher + .match_list_indices(&[display_line]) + .into_iter() + .next() + else { + continue; + }; + match_indices = re_indices; + } + + match_indices.indices.sort_unstable(); + + // Minimum matched chars: at least (needle_len - max_typos) + // characters must appear. This is consistent with the typo + // budget: each typo can drop one needle char from the alignment. + let min_matched = needle_len.saturating_sub(max_typos).max(1); + if match_indices.indices.len() < min_matched { + continue; + } + + let indices = &match_indices.indices; + + if let (Some(&first), Some(&last)) = (indices.first(), indices.last()) { + // reject widely scattered matches + let span = last - first + 1; + if span > max_match_span { + continue; + } + + // Density check: matched chars / span must be dense enough. + // Relaxed for perfect subsequence matches (all needle chars + // present), slightly relaxed for typo matches to handle + // delimiter-heavy targets + // (e.g. "ff_flv_encode_picture_header" has span inflated by underscores w/ density ~68%) + let density = (indices.len() * 100) / span; + let min_density = if indices.len() >= needle_len { + 45 // Perfect subsequence relaxed (delimiters inflate span) + } else { + 65 // Has typos filter out a long string + }; + if density < min_density { + continue; + } + + // Gap count check: count discontinuities in the indices + let gap_count = indices.windows(2).filter(|w| w[1] != w[0] + 1).count(); + if gap_count > max_gaps { + continue; + } + } + + let (ln, bo) = line_meta[idx]; + let match_byte_offsets = + char_indices_to_byte_offsets(display_line, &match_indices.indices); + let col = match_byte_offsets + .first() + .map(|r| r.0 as usize) + .unwrap_or(0); + + file_matches.push(GrepMatch { + file_index: 0, + line_number: ln, + col, + byte_offset: bo, + is_definition: classify_definition( + options.classify_definitions, + display_line, + ), + line_content: display_line.to_string(), + match_byte_offsets, + fuzzy_score: Some(match_indices.score), + context_before: Vec::new(), + context_after: Vec::new(), + }); + + if max_matches_per_file != 0 && file_matches.len() >= max_matches_per_file { + break; + } + } + + if file_matches.is_empty() { + return None; + } + + Some((chunk_offset + local_idx, *file, file_matches)) + }, + ) + .flatten() + .collect(); + + for result in chunk_results { + running_matches += result.2.len(); + per_file_results.push(result); + } + + if running_matches >= page_limit || budget_exceeded.load(Ordering::Relaxed) { + break; + } + } + + GrepResult::collect( + per_file_results, + files_to_search.len(), + options, + total_files, + filtered_file_count, + budget_exceeded.load(Ordering::Relaxed), + ) +} diff --git a/crates/fff-core/src/grep/grep.rs b/crates/fff-core/src/grep/grep.rs new file mode 100644 index 0000000..27f72bd --- /dev/null +++ b/crates/fff-core/src/grep/grep.rs @@ -0,0 +1,1548 @@ +use crate::{ + bigram_filter::{BigramFilter, BigramOverlay, extract_bigrams}, + bigram_query::{fuzzy_to_bigram_query, regex_to_bigram_query}, + constraints::{ConstraintPlan, ConstraintsBuffers}, + simd_string_utils::memmem, + sort_buffer::sort_with_buffer, + types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot}, +}; +use aho_corasick::AhoCorasick; +use fff_grep::{ + Searcher, SearcherBuilder, Sink, SinkMatch, + matcher::{Match, Matcher, NoError}, +}; +use fff_query_parser::{Constraint, FFFQuery, GrepConfig, QueryParser}; +use rayon::prelude::*; +use smallvec::SmallVec; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use tracing::Level; + +use super::utils::{GrepResult, strip_line_terminators}; + +#[cfg(feature = "definitions")] +#[inline] +pub(super) fn classify_definition(enabled: bool, line: &str) -> bool { + enabled && super::classify::is_definition_line(line) +} + +#[cfg(not(feature = "definitions"))] +#[inline] +pub(super) fn classify_definition(_enabled: bool, _line: &str) -> bool { + false +} + +/// Check if `text` contains `\n` that is NOT preceded by another `\`. +/// +/// `\n` -> true (user wants multiline search) +/// `\\n` -> false (escaped backslash followed by literal `n`, e.g. `\\nvim-data`) +#[inline] +pub(super) fn has_unescaped_newline_escape(text: &str) -> bool { + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len().saturating_sub(1) { + if bytes[i] == b'\\' { + if bytes[i + 1] == b'n' { + // Count consecutive backslashes ending at position i + let mut backslash_count = 1; + while backslash_count <= i && bytes[i - backslash_count] == b'\\' { + backslash_count += 1; + } + // Odd number of backslashes before 'n' -> real \n escape + if backslash_count % 2 == 1 { + return true; + } + } + // Skip past the escaped character + i += 2; + } else { + i += 1; + } + } + false +} + +/// Replace only unescaped `\n` sequences with real newlines. +/// +/// `\n` -> newline character +/// `\\n` -> preserved as-is (literal backslash + `n`) +pub(super) fn replace_unescaped_newline_escapes(text: &str) -> String { + let bytes = text.as_bytes(); + let mut result = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' && i + 1 < bytes.len() { + if bytes[i + 1] == b'n' { + let mut backslash_count = 1; + while backslash_count <= i && bytes[i - backslash_count] == b'\\' { + backslash_count += 1; + } + if backslash_count % 2 == 1 { + result.push(b'\n'); + i += 2; + continue; + } + } + result.push(bytes[i]); + i += 1; + } else { + result.push(bytes[i]); + i += 1; + } + } + String::from_utf8(result).unwrap_or_else(|_| text.to_string()) +} + +/// Controls how the grep pattern is interpreted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum GrepMode { + /// Literal plain text match: default path that doesn't require any regex machinery + #[default] + PlainText, + /// Regex mode: uses the same exact matching engine as ripgrep + Regex, + /// Smart fuzzy mode, allows user to make either a couple of single char typos or long gaps + /// e.g. shcema -> shcema, or UserController -> UserAuthController + /// + /// Significatnly slower than plain text, especially on unindexed FilePicker + Fuzzy, +} + +/// A single content match within a file +#[derive(Debug, Clone)] +pub struct GrepMatch { + /// Index into the deduplicated `files` vec of the GrepResult. + pub file_index: usize, + /// 1-based line number. + pub line_number: u64, + /// 0-based byte column of first match start within the line. + pub col: usize, + /// Absolute byte offset of the matched line from the start of the file. + /// Can be used by the preview to seek directly without scanning from the top. + pub byte_offset: u64, + /// The matched line text, truncated to `MAX_LINE_DISPLAY_LEN`. + pub line_content: String, + /// Byte offsets `(start, end)` within `line_content` for each match. + /// Stack-allocated for the common case of ≤4 spans per line. + pub match_byte_offsets: SmallVec<[(u32, u32); 4]>, + /// Fuzzy match score from neo_frizbee (only set in Fuzzy grep mode). + pub fuzzy_score: Option, + /// Whether the matched line looks like a definition (struct, fn, class, etc.). + /// Computed at match time so output formatters don't need to re-scan. + pub is_definition: bool, + /// Lines before the match (for context display). Empty when context is 0. + pub context_before: Vec, + /// Lines after the match (for context display). Empty when context is 0. + pub context_after: Vec, +} + +impl GrepMatch { + /// Strip leading whitespace from `line_content` and all context lines, + /// adjusting `col` and `match_byte_offsets` so highlights remain correct. + pub fn trim_leading_whitespace(&mut self) { + let strip_len = self.line_content.len() - self.line_content.trim_start().len(); + if strip_len > 0 { + self.line_content.drain(..strip_len); + let off = strip_len as u32; + self.col = self.col.saturating_sub(strip_len); + for range in &mut self.match_byte_offsets { + range.0 = range.0.saturating_sub(off); + range.1 = range.1.saturating_sub(off); + } + } + for line in &mut self.context_before { + let n = line.len() - line.trim_start().len(); + if n > 0 { + line.drain(..n); + } + } + for line in &mut self.context_after { + let n = line.len() - line.trim_start().len(); + if n > 0 { + line.drain(..n); + } + } + } +} + +pub use crate::constants::MAX_FFFILE_SIZE; + +/// Options for grep search. +#[derive(Debug, Clone)] +pub struct GrepSearchOptions { + pub max_file_size: u64, + pub max_matches_per_file: usize, + pub smart_case: bool, + /// File-based pagination offset: index into the sorted/filtered file list + /// to start searching from. Pass 0 for the first page, then use + /// `GrepResult::next_file_offset` for subsequent pages. + pub file_offset: usize, + /// Maximum number of matches to collect before stopping. + pub page_limit: usize, + /// How to interpret the search pattern. Defaults to `PlainText`. + pub mode: GrepMode, + /// Maximum time in milliseconds to spend searching before returning partial + /// results. Prevents UI freezes on pathological queries. 0 = no limit. + pub time_budget_ms: u64, + /// Number of context lines to include before each match. 0 = disabled. + pub before_context: usize, + /// Number of context lines to include after each match. 0 = disabled. + pub after_context: usize, + /// Whether to classify each match as a definition line. Adds ~2% overhead + /// on large repos; disable for interactive grep where it is not needed. + pub classify_definitions: bool, + /// Strip leading whitespace from matched lines and context lines, adjusting + /// highlight byte offsets accordingly. Useful for AI/MCP consumers and UIs + /// that don't need indentation. Default: false. + pub trim_whitespace: bool, + /// External abort signal. When provided, overrides the picker's internal + /// cancellation flag. Set to `true` to stop the search early and return + /// partial results. Omit (or use `..Default::default()`) to let the + /// picker manage cancellation. + pub abort_signal: Option>, +} + +impl Default for GrepSearchOptions { + fn default() -> Self { + Self { + max_file_size: MAX_FFFILE_SIZE, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 50, + mode: GrepMode::default(), + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } + } +} + +#[derive(Clone, Copy)] +struct GrepContext<'a, 'b> { + total_files: usize, + filtered_file_count: usize, + budget: &'a ContentCacheBudget, + base_path: &'a Path, + arena: crate::simd_path::ArenaPtr, + overflow_arena: crate::simd_path::ArenaPtr, + prefilter: Option<&'a memchr::memmem::Finder<'b>>, + prefilter_case_insensitive: bool, + abort_signal: &'a AtomicBool, +} + +impl GrepContext<'_, '_> { + #[inline] + fn arena_for_file(&self, file: &FileItem) -> crate::simd_path::ArenaPtr { + if file.is_overflow() { + self.overflow_arena + } else { + self.arena + } + } +} + +struct RegexMatcher<'r> { + regex: &'r regex::bytes::Regex, + is_multiline: bool, +} + +impl Matcher for RegexMatcher<'_> { + type Error = NoError; + + #[inline] + fn find_at(&self, haystack: &[u8], at: usize) -> Result, NoError> { + Ok(self + .regex + .find_at(haystack, at) + .map(|m| Match::new(m.start(), m.end()))) + } + + #[inline] + fn line_terminator(&self) -> Option { + if self.is_multiline { + None + } else { + Some(fff_grep::LineTerminator::byte(b'\n')) + } + } +} + +/// A `grep_matcher::Matcher` backed by `memchr::memmem` for literal search. +/// +/// This is used in `PlainText` mode and is significantly faster than regex +/// for literal patterns: memchr uses SIMD (AVX2/NEON) two-way substring +/// search internally, avoiding the overhead of regex compilation and DFA +/// state transitions. +/// +/// Always reports `\n` as line terminator so the searcher uses the fast +/// candidate-line path (plain text can never span lines unless `\n` is +/// literally in the needle, which we handle separately). +struct PlainTextMatcher<'a> { + /// Case-folded needle bytes for case-insensitive matching. + /// When case-sensitive, this is the original pattern bytes. + needle: &'a [u8], + case_insensitive: bool, +} + +impl Matcher for PlainTextMatcher<'_> { + type Error = NoError; + + #[inline] + fn find_at(&self, haystack: &[u8], at: usize) -> Result, NoError> { + let hay = &haystack[at..]; + + let found = if self.case_insensitive { + memmem::find(hay, self.needle) + } else { + memchr::memmem::find(hay, self.needle) + }; + + Ok(found.map(|pos| Match::new(at + pos, at + pos + self.needle.len()))) + } + + #[inline] + fn line_terminator(&self) -> Option { + Some(fff_grep::LineTerminator::byte(b'\n')) + } +} + +/// Maximum bytes of a matched line to keep for display. Prevents minified +/// JS or huge single-line files from blowing up memory. +const MAX_LINE_DISPLAY_LEN: usize = 512; + +struct SinkState { + file_index: usize, + matches: Vec, + max_matches: usize, + before_context: usize, + after_context: usize, + classify_definitions: bool, +} + +impl SinkState { + #[inline] + fn prepare_line<'a>(line_bytes: &'a [u8], mat: &SinkMatch<'_>) -> (&'a [u8], u32, u64, u64) { + let line_number = mat.line_number().unwrap_or(0); + let byte_offset = mat.absolute_byte_offset(); + + // Trim trailing newline/CR directly on bytes to avoid UTF-8 conversion. + let trimmed_bytes = strip_line_terminators(line_bytes); + + // Truncate for display (floor to a char boundary). + let display_bytes = truncate_display_bytes(trimmed_bytes); + + let display_len = display_bytes.len() as u32; + (display_bytes, display_len, line_number, byte_offset) + } + + #[inline] + #[allow(clippy::too_many_arguments)] + fn push_match( + &mut self, + line_number: u64, + col: usize, + byte_offset: u64, + line_content: String, + match_byte_offsets: SmallVec<[(u32, u32); 4]>, + context_before: Vec, + context_after: Vec, + ) { + let is_definition = classify_definition(self.classify_definitions, &line_content); + self.matches.push(GrepMatch { + file_index: self.file_index, + line_number, + col, + byte_offset, + line_content, + match_byte_offsets, + fuzzy_score: None, + is_definition, + context_before, + context_after, + }); + } + + /// Extract context lines from the full buffer around a matched region. + fn extract_context(&self, mat: &SinkMatch<'_>) -> (Vec, Vec) { + if self.before_context == 0 && self.after_context == 0 { + return (Vec::new(), Vec::new()); + } + + let buffer = mat.buffer(); + let range = mat.bytes_range_in_buffer(); + + let mut before = Vec::new(); + if self.before_context > 0 && range.start > 0 { + // Walk backward from the start of the match line to find preceding lines + let mut pos = range.start; + let mut lines_found = 0; + while lines_found < self.before_context && pos > 0 { + // Skip the newline just before our current position + pos -= 1; + // Find the previous newline + let line_start = match memchr::memrchr(b'\n', &buffer[..pos]) { + Some(nl) => nl + 1, + None => 0, + }; + let line = &buffer[line_start..pos]; + // Trim trailing \r + let line = if line.last() == Some(&b'\r') { + &line[..line.len() - 1] + } else { + line + }; + let truncated = truncate_display_bytes(line); + before.push(String::from_utf8_lossy(truncated).into_owned()); + pos = line_start; + lines_found += 1; + } + before.reverse(); + } + + let mut after = Vec::new(); + if self.after_context > 0 && range.end < buffer.len() { + let mut pos = range.end; + let mut lines_found = 0; + while lines_found < self.after_context && pos < buffer.len() { + // Find the next newline + let line_end = match memchr::memchr(b'\n', &buffer[pos..]) { + Some(nl) => pos + nl, + None => buffer.len(), + }; + let line = &buffer[pos..line_end]; + // Trim trailing \r + let line = if line.last() == Some(&b'\r') { + &line[..line.len() - 1] + } else { + line + }; + let truncated = truncate_display_bytes(line); + after.push(String::from_utf8_lossy(truncated).into_owned()); + pos = if line_end < buffer.len() { + line_end + 1 // skip past \n + } else { + buffer.len() + }; + lines_found += 1; + } + } + + (before, after) + } +} + +/// Truncate a byte slice for display, respecting UTF-8 char boundaries. +#[inline] +pub(super) fn truncate_display_bytes(bytes: &[u8]) -> &[u8] { + if bytes.len() <= MAX_LINE_DISPLAY_LEN { + bytes + } else { + let mut end = MAX_LINE_DISPLAY_LEN; + while end > 0 && !is_utf8_char_boundary(bytes[end]) { + end -= 1; + } + &bytes[..end] + } +} + +/// Sink for `PlainText` mode. +/// +/// Highlights are extracted with `memchr::memmem::Finder` (case-sensitive) +/// or the SIMD `simd_string_utils::memmem` search (case-insensitive). No regex engine is +/// involved at any point. +struct PlainTextSink<'r> { + state: SinkState, + finder: &'r memchr::memmem::Finder<'r>, + pattern_len: u32, + case_insensitive: bool, +} + +impl Sink for PlainTextSink<'_> { + type Error = std::io::Error; + + fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result { + if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches { + return Ok(false); + } + + let line_bytes = mat.bytes(); + let (display_bytes, display_len, line_number, byte_offset) = + SinkState::prepare_line(line_bytes, mat); + + let line_content = String::from_utf8_lossy(display_bytes).into_owned(); + let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new(); + let mut col = 0usize; + let mut first = true; + + if self.case_insensitive { + // The finder was built over the lowered pattern, so its needle is + // exactly the `needle_lower` expected by `memmem::find`. + let needle_lower = self.finder.needle(); + let mut start_pos = 0usize; + while let Some(pos) = memmem::find(&display_bytes[start_pos..], needle_lower) { + let abs_start = (start_pos + pos) as u32; + let abs_end = (abs_start + self.pattern_len).min(display_len); + if first { + col = abs_start as usize; + first = false; + } + match_byte_offsets.push((abs_start, abs_end)); + start_pos += pos + 1; + } + } else { + let mut start_pos = 0usize; + while let Some(pos) = self.finder.find(&display_bytes[start_pos..]) { + let abs_start = (start_pos + pos) as u32; + let abs_end = (abs_start + self.pattern_len).min(display_len); + if first { + col = abs_start as usize; + first = false; + } + match_byte_offsets.push((abs_start, abs_end)); + start_pos += pos + 1; + } + } + + let (context_before, context_after) = self.state.extract_context(mat); + self.state.push_match( + line_number, + col, + byte_offset, + line_content, + match_byte_offsets, + context_before, + context_after, + ); + Ok(true) + } + + fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> { + Ok(()) + } +} + +/// Sink for `Regex` mode. +/// +/// Uses the compiled regex to extract precise variable-length highlight spans +/// from each matched line. No `memmem` finder is involved. +struct RegexSink<'r> { + state: SinkState, + re: &'r regex::bytes::Regex, +} + +impl Sink for RegexSink<'_> { + type Error = std::io::Error; + + fn matched( + &mut self, + _searcher: &Searcher, + sink_match: &SinkMatch<'_>, + ) -> Result { + if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches { + return Ok(false); + } + + let line_bytes = sink_match.bytes(); + let (display_bytes, display_len, line_number, byte_offset) = + SinkState::prepare_line(line_bytes, sink_match); + + let line_content = String::from_utf8_lossy(display_bytes).into_owned(); + let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new(); + let mut col = 0usize; + let mut first = true; + + for m in self.re.find_iter(display_bytes) { + let abs_start = m.start() as u32; + let abs_end = (m.end() as u32).min(display_len); + if first { + col = abs_start as usize; + first = false; + } + match_byte_offsets.push((abs_start, abs_end)); + } + + let (context_before, context_after) = self.state.extract_context(sink_match); + self.state.push_match( + line_number, + col, + byte_offset, + line_content, + match_byte_offsets, + context_before, + context_after, + ); + Ok(true) + } + + fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> { + Ok(()) + } +} + +/// A `grep_matcher::Matcher` backed by Aho-Corasick for multi-pattern search. +/// +/// Finds the first occurrence of any pattern starting at the given offset. +/// Always reports `\n` as the line terminator for the fast candidate-line path. +struct AhoCorasickMatcher<'a> { + ac: &'a AhoCorasick, +} + +impl Matcher for AhoCorasickMatcher<'_> { + type Error = NoError; + + #[inline] + fn find_at(&self, haystack: &[u8], at: usize) -> std::result::Result, NoError> { + let hay = &haystack[at..]; + let found: Option = self.ac.find(hay); + Ok(found.map(|m| Match::new(at + m.start(), at + m.end()))) + } + + #[inline] + fn line_terminator(&self) -> Option { + Some(fff_grep::LineTerminator::byte(b'\n')) + } +} + +/// Sink for Aho-Corasick multi-pattern mode. +/// +/// Collects all pattern match positions on each matched line for highlighting. +struct AhoCorasickSink<'a> { + state: SinkState, + ac: &'a AhoCorasick, +} + +impl Sink for AhoCorasickSink<'_> { + type Error = std::io::Error; + + fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result { + if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches { + return Ok(false); + } + + let line_bytes = mat.bytes(); + let (display_bytes, display_len, line_number, byte_offset) = + SinkState::prepare_line(line_bytes, mat); + + let line_content = String::from_utf8_lossy(display_bytes).into_owned(); + let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new(); + let mut col = 0usize; + let mut first = true; + + for m in self.ac.find_iter(display_bytes as &[u8]) { + let abs_start = m.start() as u32; + let abs_end = (m.end() as u32).min(display_len); + if first { + col = abs_start as usize; + first = false; + } + match_byte_offsets.push((abs_start, abs_end)); + } + + let (context_before, context_after) = self.state.extract_context(mat); + self.state.push_match( + line_number, + col, + byte_offset, + line_content, + match_byte_offsets, + context_before, + context_after, + ); + Ok(true) + } + + fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> { + Ok(()) + } +} + +/// Multi-pattern OR search using Aho-Corasick. +/// +/// Builds a single automaton from all patterns and searches each file in one +/// pass. This is significantly faster than regex alternation for literal text +/// searches because Aho-Corasick uses SIMD-accelerated multi-needle matching. +/// +/// Returns the same `GrepResult` type as `grep_search`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn multi_grep_search<'a>( + files: &'a [FileItem], + patterns: &[&str], + constraints: &[fff_query_parser::Constraint<'_>], + options: &GrepSearchOptions, + budget: &ContentCacheBudget, + bigram_index: Option<&BigramFilter>, + bigram_overlay: Option<&BigramOverlay>, + abort_signal: &AtomicBool, + base_path: &Path, + arena: crate::simd_path::ArenaPtr, + overflow_arena: crate::simd_path::ArenaPtr, +) -> GrepResult<'a> { + let total_files = files.live_count(); + + if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) { + return GrepResult::empty(total_files, total_files); + } + + // Bigram prefiltering: OR the candidate bitsets for each pattern. + // A file is a candidate if it matches ANY of the patterns' bigrams. + let bigram_candidates = if let Some(idx) = bigram_index + && idx.is_ready() + { + let mut combined: Option> = None; + for pattern in patterns { + if let Some(candidates) = idx.query(pattern.as_bytes()) { + combined = Some(match combined { + None => candidates, + Some(mut acc) => { + // OR: file is candidate if it matches any pattern + acc.iter_mut() + .zip(candidates.iter()) + .for_each(|(a, b)| *a |= *b); + acc + } + }); + } + } + + if let Some(ref mut candidates) = combined + && let Some(overlay) = bigram_overlay + { + for pattern in patterns { + let pattern_bigrams = extract_bigrams(pattern.as_bytes()); + for file_idx in overlay.query_modified(&pattern_bigrams) { + let word = file_idx / 64; + if word < candidates.len() { + candidates[word] |= 1u64 << (file_idx % 64); + } + } + } + } + + combined + } else { + None + }; + + let base_file_count = match bigram_overlay { + Some(bigram_overlay) => bigram_overlay.base_file_count(), + None => files.len(), + }; + + let (mut files_to_search, mut filtered_file_count) = prefilter_files( + files, + constraints, + bigram_candidates.as_deref(), + base_file_count, + options, + arena, + overflow_arena, + ); + + // If constraints yielded 0 files and we had FilePath constraints, + // retry without them (the path token was likely part of the search text). + if files_to_search.is_empty() + && let Some(stripped) = strip_file_path_constraint_if_present(constraints) + { + let (retry_files, retry_count) = prefilter_files( + files, + &stripped, + bigram_candidates.as_deref(), + base_file_count, + options, + arena, + overflow_arena, + ); + files_to_search = retry_files; + filtered_file_count = retry_count; + } + + if files_to_search.is_empty() { + return GrepResult::empty(total_files, filtered_file_count); + } + + // Smart case: case-insensitive when all patterns are lowercase + let case_insensitive = if options.smart_case { + !patterns.iter().any(|p| p.chars().any(|c| c.is_uppercase())) + } else { + false + }; + + let ac = aho_corasick::AhoCorasickBuilder::new() + .ascii_case_insensitive(case_insensitive) + .build(patterns) + .expect("Aho-Corasick build should not fail for literal patterns"); + + let searcher = { + let mut b = SearcherBuilder::new(); + b.line_number(true); + b + } + .build(); + + let ac_matcher = AhoCorasickMatcher { ac: &ac }; + perform_grep( + &files_to_search, + options, + &GrepContext { + total_files, + filtered_file_count, + budget, + base_path, + arena, + overflow_arena, + prefilter: None, // no memmem prefilter for multi-pattern search + prefilter_case_insensitive: false, + abort_signal, + }, + |file_bytes: &[u8], max_matches: usize| { + let state = SinkState { + file_index: 0, + matches: Vec::with_capacity(4), + max_matches, + before_context: options.before_context, + after_context: options.after_context, + classify_definitions: options.classify_definitions, + }; + + let mut sink = AhoCorasickSink { state, ac: &ac }; + + if let Err(e) = searcher.search_slice(&ac_matcher, file_bytes, &mut sink) { + tracing::error!(error = %e, "Grep (aho-corasick multi) search failed"); + } + + sink.state.matches + }, + ) +} + +// copied from the rust u8 private method +#[inline] +const fn is_utf8_char_boundary(b: u8) -> bool { + (b as i8) >= -0x40 +} + +fn build_regex(pattern: &str, smart_case: bool) -> Result { + if pattern.is_empty() { + return Err("empty pattern".to_string()); + } + + let regex_pattern = if pattern.contains("\\n") { + pattern.replace("\\n", "\n") + } else { + pattern.to_string() + }; + + let case_insensitive = if smart_case { + !pattern.chars().any(|c| c.is_uppercase()) + } else { + false + }; + + regex::bytes::RegexBuilder::new(®ex_pattern) + .case_insensitive(case_insensitive) + .multi_line(true) + .unicode(false) + .build() + .map_err(|e| e.to_string()) +} + +/// Convert character-position indices from neo_frizbee into byte-offset +/// pairs (start, end) suitable for `match_byte_offsets`. +/// +/// frizbee returns character positions (0-based index into the char +/// iterator). We need byte ranges because the UI renderer and Lua layer +/// use byte offsets for extmark highlights. +/// +/// Each matched character becomes its own (byte_start, byte_end) pair. +/// Adjacent characters are merged into a single contiguous range. +pub(super) fn char_indices_to_byte_offsets( + line: &str, + char_indices: &[usize], +) -> SmallVec<[(u32, u32); 4]> { + if char_indices.is_empty() { + return SmallVec::new(); + } + + // Build a map: char_index -> (byte_start, byte_end) for all chars. + // Iterating all chars is O(n) in the line length which is bounded by MAX_LINE_DISPLAY_LEN (512). + let char_byte_ranges: Vec<(usize, usize)> = line + .char_indices() + .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8())) + .collect(); + + // Convert char indices to byte ranges, merging adjacent ranges + let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len()); + + for &ci in char_indices { + if ci >= char_byte_ranges.len() { + continue; // out of bounds (shouldn't happen with valid data) + } + let (start, end) = char_byte_ranges[ci]; + // Merge with previous range if adjacent + if let Some(last) = result.last_mut() + && last.1 == start as u32 + { + last.1 = end as u32; + continue; + } + result.push((start as u32, end as u32)); + } + + result +} + +#[tracing::instrument( + skip_all, + level = Level::DEBUG, + fields(prefiltered_count = files_to_search.len()) +)] +fn perform_grep<'a, F>( + files_to_search: &[&'a FileItem], + options: &GrepSearchOptions, + ctx: &GrepContext<'_, '_>, + search_file: F, +) -> GrepResult<'a> +where + F: Fn(&[u8], usize) -> Vec + Sync, +{ + let time_budget = if options.time_budget_ms > 0 { + Some(std::time::Duration::from_millis(options.time_budget_ms)) + } else { + None + }; + + let search_start = std::time::Instant::now(); + let page_limit = options.page_limit; + let budget_exceeded = AtomicBool::new(false); + + let mut result_files: Vec<&'a FileItem> = Vec::new(); + let mut all_matches: Vec = Vec::new(); + let mut files_consumed: usize = 0; + let mut page_filled = false; + + // Each chunk is a rayon barrier. A flat small chunk over 500k files = ~7800 + // barriers; x2 growth makes it logarithmic. But a too-aggressive growth + // over-scans: when a page fills mid-chunk, the whole submitted chunk still + // runs. + // + // So only grow when the prefilter is weak (large candidate set); + // when bigram cut the set in half, keep fixed small chunks for cheap page-fill termination. + let base_chunk = rayon::current_num_threads() * 4; + let prefilter_strong = ctx.total_files > 0 && files_to_search.len() * 2 < ctx.total_files; + let max_chunk = if prefilter_strong { + base_chunk + } else { + (base_chunk * 256).max(8 * 1024) + }; + let growth = if prefilter_strong { 1 } else { 2 }; + let mut chunk_size = base_chunk; + let mut chunk_start = 0; + + while chunk_start < files_to_search.len() { + let chunk_end = (chunk_start + chunk_size).min(files_to_search.len()); + let chunk = &files_to_search[chunk_start..chunk_end]; + chunk_start = chunk_end; + chunk_size = (chunk_size * growth).min(max_chunk); + let chunk_offset = files_consumed; + + let chunk_results: Vec<(usize, &'a FileItem, Vec)> = chunk + .par_iter() + .enumerate() + .map_init( + // tested it out a few times, this is just fine for rayon worker in this specific + // case it doesn't reallocate this many times and it is actually faster than using + // scoped threads with a predefined local scratch buffers because of spawn cost + || (Vec::with_capacity(64 * 1024), MmapSlot::default()), + |(buf, mmap_slot), (local_idx, file)| { + // perform all the atomic machinery on every 8th + if local_idx % 8 == 0 { + let mut need_abort = ctx.abort_signal.load(Ordering::Relaxed); + if !need_abort + && let Some(budget) = time_budget + && all_matches.len() > 1 + && search_start.elapsed() > budget + { + need_abort = true; + } + + if need_abort { + budget_exceeded.store(true, Ordering::Relaxed); + return None; + } + } + + let content = file.get_content_for_search( + buf, + mmap_slot, + ctx.arena_for_file(file), + ctx.base_path, + ctx.budget, + )?; + + // Fast whole-file memmem check before entering the + // grep-searcher machinery. Skips Vec alloc, Searcher + // setup, and line-splitting for files that can't match. + if let Some(pf) = ctx.prefilter { + let found = if ctx.prefilter_case_insensitive { + memmem::find(content, pf.needle()).is_some() + } else { + pf.find(content).is_some() + }; + if !found { + return None; + } + } + + let file_matches = search_file(content, options.max_matches_per_file); + + if file_matches.is_empty() { + return None; + } + + Some((chunk_offset + local_idx, *file, file_matches)) + }, + ) + .flatten() + .collect(); + + // Every file in the chunk was visited by rayon (matched or not). + files_consumed = chunk_offset + chunk.len(); + + // Flatten this chunk's results into the accumulator. + for (batch_idx, file, file_matches) in chunk_results { + let file_result_idx = result_files.len(); + result_files.push(file); + + for mut m in file_matches { + m.file_index = file_result_idx; + if options.trim_whitespace { + m.trim_leading_whitespace(); + } + all_matches.push(m); + } + + if all_matches.len() >= page_limit { + // Tighten files_consumed to the file that tipped us over so + // the next page resumes right after it. + files_consumed = batch_idx + 1; + page_filled = true; + break; + } + } + + if page_filled || budget_exceeded.load(Ordering::Relaxed) { + break; + } + } + + // If no file had any match, we searched the entire slice. + if result_files.is_empty() { + files_consumed = files_to_search.len(); + } + + let has_more = budget_exceeded.load(Ordering::Relaxed) + || (page_filled && files_consumed < files_to_search.len()); + + let next_file_offset = if has_more { + options.file_offset + files_consumed + } else { + 0 + }; + + GrepResult { + matches: all_matches, + files_with_matches: result_files.len(), + files: result_files, + total_files_searched: files_consumed, + total_files: ctx.total_files, + filtered_file_count: ctx.filtered_file_count, + next_file_offset, + regex_fallback_error: None, + } +} + +/// Single pass prefilter that doesn't involve file reading +/// allocates only amount of memory required for storing references of the FileItems have to be +/// opened for grepping unaviodably, in the worst case allocates N * memory if no prefilter needed +fn prefilter_files<'a>( + files: &'a [FileItem], + constraints: &[fff_query_parser::Constraint<'_>], + bigram_candidates: Option<&[u64]>, + base_count: usize, + options: &GrepSearchOptions, + arena: crate::simd_path::ArenaPtr, + overflow_arena: crate::simd_path::ArenaPtr, +) -> (Vec<&'a FileItem>, usize) { + let max_file_size = options.max_file_size; + let plan = if constraints.is_empty() { + None + } else { + Some(ConstraintPlan::build( + constraints, + files, + arena, + overflow_arena, + )) + }; + + let mut scratch = ConstraintsBuffers::new(); + + #[inline(always)] + fn basic_prefilter(file: &FileItem, max: u64) -> bool { + !file.is_deleted() && !file.is_binary() && file.size > 0 && file.size <= max + } + + // squeeze as much prefilters into a single loop as possible + let mut prefiltered: Vec<&FileItem> = match bigram_candidates { + Some(candidates) => { + let boundary = base_count.min(files.len()); + let (indexed, tail) = files.split_at(boundary); + + let cap = BigramFilter::count_candidates(candidates) + tail.len(); + let mut out: Vec<&FileItem> = Vec::with_capacity(cap); + + let full_words = boundary / 64; + let last_word_bits = boundary % 64; + + // we need this because we already had a regression of the wrong bit + // has been set for the very last word based on the overlay, it's pretty cheap + macro_rules! evaluate_bigram_match_word { + ($word:expr, $base:expr) => {{ + let mut bits: u64 = $word; + while bits != 0 { + let bit = bits.trailing_zeros() as usize; + let file_idx = $base + bit; + bits &= bits - 1; + + let f = unsafe { indexed.get_unchecked(file_idx) }; + if !basic_prefilter(f, max_file_size) { + continue; + } + if let Some(plan) = plan.as_ref() + && !plan.matches(f, file_idx, arena, overflow_arena, &mut scratch) + { + continue; + } + out.push(f); + } + }}; + } + + // Full words: every set bit guaranteed `< boundary`. + for (word_idx, &word) in candidates.iter().take(full_words).enumerate() { + if word != 0 { + evaluate_bigram_match_word!(word, word_idx * 64); + } + } + + // Last partial word: mask bits past `boundary` once at word load. + if last_word_bits != 0 { + // this will get only (mod 64) bits from the last word guaratee that it's 0 padded + let last_mask: u64 = (1u64 << last_word_bits) - 1; + let word = candidates[full_words] & last_mask; + if word != 0 { + evaluate_bigram_match_word!(word, full_words * 64); + } + } + + // Sequential processing for non-bigrammable files: they are always in the end + for (offset, f) in tail.iter().enumerate() { + if !basic_prefilter(f, max_file_size) { + continue; + } + if let Some(ref p) = plan + && !p.matches(f, boundary + offset, arena, overflow_arena, &mut scratch) + { + continue; + } + out.push(f); + } + + out + } + // this will be executed if there is no bigram, in the worst case it will allocate + // whole array of files but probability in the real repo of NO preflter working is so + // low that we just ignore that, usually there would be at least a few files excluded + None => { + let mut out: Vec<&FileItem> = Vec::new(); + for (idx, f) in files.iter().enumerate() { + if !basic_prefilter(f, max_file_size) { + continue; + } + if let Some(ref p) = plan + && !p.matches(f, idx, arena, overflow_arena, &mut scratch) + { + continue; + } + out.push(f); + } + out + } + }; + + let total_count = prefiltered.len(); + + sort_with_buffer(&mut prefiltered, |a, b| { + b.total_frecency_score() + .cmp(&a.total_frecency_score()) + .then(b.modified.cmp(&a.modified)) + }); + + if options.file_offset > 0 && options.file_offset < total_count { + let paginated = prefiltered.split_off(options.file_offset); + (paginated, total_count) + } else if options.file_offset >= total_count { + (Vec::new(), total_count) + } else { + (prefiltered, total_count) + } +} + +/// Perform a grep search across all indexed files. +/// +/// When `query` is empty, returns git-modified/untracked files sorted by +/// frecency for the "welcome state" UI. +#[tracing::instrument(skip_all, fields(file_count = files.len()))] +#[allow(clippy::too_many_arguments)] +pub(crate) fn grep_search<'a>( + files: &'a [FileItem], + query: &FFFQuery<'_>, + options: &GrepSearchOptions, + budget: &ContentCacheBudget, + bigram_index: Option<&BigramFilter>, + bigram_overlay: Option<&BigramOverlay>, + abort_signal: &AtomicBool, + base_path: &Path, + arena: crate::simd_path::ArenaPtr, + overflow_arena: crate::simd_path::ArenaPtr, +) -> GrepResult<'a> { + let total_files = files.live_count(); + + // Extract the grep text and file constraints from the parsed query. + // For grep, the search pattern is the original query with constraint tokens + // removed. All non-constraint text tokens are collected and joined with + // spaces to form the grep pattern: + // "name = *.rs someth" -> grep "name = someth" with constraint Extension("rs") + let constraints_from_query = &query.constraints[..]; + + let grep_text = if !matches!(query.fuzzy_query, fff_query_parser::FuzzyQuery::Empty) { + query.grep_text() + } else { + // if constraint-only or empty query we use raw_query for backslash-escape handling + let t = query.raw_query.trim(); + if t.starts_with('\\') && t.len() > 1 { + let suffix = &t[1..]; + let parser = QueryParser::new(GrepConfig); + if !parser.parse(suffix).constraints.is_empty() { + suffix.to_string() + } else { + t.to_string() + } + } else { + t.to_string() + } + }; + + if grep_text.is_empty() { + return GrepResult::empty(total_files, total_files); + } + + let case_insensitive = if options.smart_case { + !grep_text.chars().any(|c| c.is_uppercase()) + } else { + false + }; + + let mut regex_fallback_error: Option = None; + let regex = match options.mode { + GrepMode::PlainText => None, + GrepMode::Fuzzy => { + // Bigram prefilter: pick 5 evenly-spaced probe bigrams, require + // (5 - max_typos) of them to appear. Widely-spaced probes are + // far more selective than sliding windows of adjacent bigrams. + let bigram_candidates = if let Some(idx) = bigram_index + && idx.is_ready() + { + let bq = fuzzy_to_bigram_query(&grep_text, 7); + if !bq.is_any() + && let Some(mut candidates) = bq.evaluate(idx) + { + if let Some(overlay) = bigram_overlay { + for (r, t) in candidates.iter_mut().zip(overlay.tombstones().iter()) { + *r &= !t; + } + // Fuzzy: conservatively add all modified files + for file_idx in overlay.modified_indices() { + let word = file_idx / 64; + if word < candidates.len() { + candidates[word] |= 1u64 << (file_idx % 64); + } + } + } + Some(candidates) + } else { + None + } + } else { + None + }; + + let base_count = match bigram_overlay { + Some(bigram_overlay) => bigram_overlay.base_file_count(), + None => files.len(), + }; + + let (mut files_to_search, mut filtered_file_count) = prefilter_files( + files, + constraints_from_query, + bigram_candidates.as_deref(), + base_count, + options, + arena, + overflow_arena, + ); + + if files_to_search.is_empty() + && let Some(stripped) = + strip_file_path_constraint_if_present(constraints_from_query) + { + let (retry_files, retry_count) = prefilter_files( + files, + &stripped, + bigram_candidates.as_deref(), + base_count, + options, + arena, + overflow_arena, + ); + + files_to_search = retry_files; + filtered_file_count = retry_count; + } + + if files_to_search.is_empty() { + return GrepResult::empty(total_files, filtered_file_count); + } + + return super::fuzzy_grep::fuzzy_grep_search( + &grep_text, + &files_to_search, + options, + total_files, + filtered_file_count, + case_insensitive, + budget, + abort_signal, + base_path, + arena, + overflow_arena, + ); + } + GrepMode::Regex => build_regex(&grep_text, options.smart_case) + .inspect_err(|err| { + tracing::warn!("Regex compilation failed for {}. Error {}", grep_text, err); + + regex_fallback_error = Some(err.to_string()); + }) + .ok(), + }; + + let is_multiline = has_unescaped_newline_escape(&grep_text); + + let effective_pattern = if is_multiline { + replace_unescaped_newline_escapes(&grep_text) + } else { + grep_text.to_string() + }; + + let finder_pattern: Vec = if case_insensitive { + effective_pattern.as_bytes().to_ascii_lowercase() + } else { + effective_pattern.as_bytes().to_vec() + }; + let finder = memchr::memmem::Finder::new(&finder_pattern); + let pattern_len = finder_pattern.len() as u32; + + // Bigram prefiltering: query the inverted index + merge overlay. + // For PlainText mode: extract bigrams directly from the literal pattern. + // For Regex mode: decompose the regex HIR into an AND/OR bigram query tree + // and evaluate it against the inverted index (supports alternation, optional + // groups, character classes, and sparse-1 bigrams across single-byte wildcards). + let bigram_candidates = if let Some(idx) = bigram_index + && idx.is_ready() + { + let raw_candidates = if regex.is_none() { + // PlainText or regex-fallback-to-plain: literal bigram query + idx.query(effective_pattern.as_bytes()) + } else { + // Regex mode: decompose pattern into bigram query tree + let bq = regex_to_bigram_query(&effective_pattern); + if !bq.is_any() { bq.evaluate(idx) } else { None } + }; + + if let Some(mut candidates) = raw_candidates { + if let Some(overlay) = bigram_overlay { + // Clear tombstoned (deleted) files from candidates + for (r, t) in candidates.iter_mut().zip(overlay.tombstones().iter()) { + *r &= !t; + } + + if regex.is_none() { + let pattern_bigrams = extract_bigrams(effective_pattern.as_bytes()); + for file_idx in overlay.query_modified(&pattern_bigrams) { + let word = file_idx / 64; + if word < candidates.len() { + candidates[word] |= 1u64 << (file_idx % 64); + } + } + } else { + for file_idx in overlay.modified_indices() { + let word = file_idx / 64; + if word < candidates.len() { + candidates[word] |= 1u64 << (file_idx % 64); + } + } + } + } + Some(candidates) + } else { + None + } + } else { + None + }; + + // Bigram bitset only covers `files[..bigram_boundary]`, new files aka overflow + // (max 1024 always scanned) + let bigram_boundary = bigram_overlay + .map(|o| o.base_file_count()) + .unwrap_or(files.len()); + + let (mut files_to_search, mut filtered_file_count) = prefilter_files( + files, + constraints_from_query, + bigram_candidates.as_deref(), + bigram_boundary, + options, + arena, + overflow_arena, + ); + + if files_to_search.is_empty() + && let Some(stripped) = strip_file_path_constraint_if_present(constraints_from_query) + { + let (retry_files, retry_count) = prefilter_files( + files, + &stripped, + bigram_candidates.as_deref(), + bigram_boundary, + options, + arena, + overflow_arena, + ); + files_to_search = retry_files; + filtered_file_count = retry_count; + } + + if files_to_search.is_empty() { + return GrepResult::empty(total_files, filtered_file_count); + } + + // `PlainTextMatcher` is used by the grep-searcher engine for line detection. + // `PlainTextSink` / `RegexSink` handle highlight extraction independently via ripgrep create + let plain_matcher = PlainTextMatcher { + needle: &finder_pattern, + case_insensitive, + }; + + let searcher = { + let mut b = SearcherBuilder::new(); + b.line_number(true).multi_line(is_multiline); + b + } + .build(); + + let should_prefilter = regex.is_none(); + let mut result = perform_grep( + &files_to_search, + options, + &GrepContext { + total_files, + filtered_file_count, + budget, + base_path, + arena, + overflow_arena, + prefilter: should_prefilter.then_some(&finder), + prefilter_case_insensitive: case_insensitive, + abort_signal, + }, + |file_bytes: &[u8], max_matches: usize| { + let state = SinkState { + file_index: 0, + matches: Vec::with_capacity(4), + max_matches, + before_context: options.before_context, + after_context: options.after_context, + classify_definitions: options.classify_definitions, + }; + + match regex { + Some(ref re) => { + let regex_matcher = RegexMatcher { + regex: re, + is_multiline, + }; + let mut sink = RegexSink { state, re }; + if let Err(e) = searcher.search_slice(®ex_matcher, file_bytes, &mut sink) { + tracing::error!(error = %e, "Grep (regex) search failed"); + } + sink.state.matches + } + None => { + let mut sink = PlainTextSink { + state, + finder: &finder, + pattern_len, + case_insensitive, + }; + if let Err(e) = searcher.search_slice(&plain_matcher, file_bytes, &mut sink) { + tracing::error!(error = %e, "Grep (plain text) search failed"); + } + sink.state.matches + } + } + }, + ); + result.regex_fallback_error = regex_fallback_error; + result +} + +pub fn parse_grep_query(query: &str) -> FFFQuery<'_> { + let parser = QueryParser::new(GrepConfig); + parser.parse(query) +} + +fn strip_file_path_constraint_if_present<'a>( + constraints: &[Constraint<'a>], +) -> Option> { + if !constraints + .iter() + .any(|c| matches!(c, Constraint::FilePath(_))) + { + return None; + } + + let filtered: fff_query_parser::ConstraintVec<'a> = constraints + .iter() + .filter(|c| !matches!(c, Constraint::FilePath(_))) + .cloned() + .collect(); + + Some(filtered) +} diff --git a/crates/fff-core/src/grep/grep_tests.rs b/crates/fff-core/src/grep/grep_tests.rs new file mode 100644 index 0000000..d342902 --- /dev/null +++ b/crates/fff-core/src/grep/grep_tests.rs @@ -0,0 +1,372 @@ +use super::grep::*; + +use crate::bigram_filter::BigramIndexBuilder; +use crate::file_picker::{FilePicker, FilePickerOptions}; +use std::io::Write; +use std::sync::atomic::AtomicBool; + +#[test] +fn test_unescaped_newline_detection() { + // Single \n → multiline + assert!(has_unescaped_newline_escape("foo\\nbar")); + // \\n → escaped backslash + literal n, NOT multiline + // (this is what the user types when grepping Rust source with `\\nvim`) + assert!(!has_unescaped_newline_escape("foo\\\\nvim-data")); + // Real-world: source file has literal \\AppData\\Local\\nvim-data + // (double backslash in the file, so user types double backslash) + assert!(!has_unescaped_newline_escape( + r#"format!("{}\\AppData\\Local\\nvim-data","# + )); + // No \n at all + assert!(!has_unescaped_newline_escape("hello world")); + // \\\\n → even number of backslashes before n → NOT multiline + assert!(!has_unescaped_newline_escape("foo\\\\\\\\nbar")); + // \\\n → 3 backslashes: first two pair up, third + n = \n → multiline + assert!(has_unescaped_newline_escape("foo\\\\\\nbar")); +} + +#[test] +fn test_replace_unescaped_newline() { + // \n → real newline + assert_eq!(replace_unescaped_newline_escapes("foo\\nbar"), "foo\nbar"); + // \\n → preserved as-is + assert_eq!( + replace_unescaped_newline_escapes("foo\\\\nvim"), + "foo\\\\nvim" + ); +} + +#[test] +fn test_fuzzy_typo_scoring() { + // Mirror the config from fuzzy_grep_search + let needle = "schema"; + let max_typos = (needle.len() / 3).min(2); // 2 + let config = neo_frizbee::Config { + max_typos: Some(max_typos as u16), + sort: false, + scoring: neo_frizbee::Scoring { + exact_match_bonus: 100, + ..neo_frizbee::Scoring::default() + }, + ..Default::default() + }; + let min_matched = needle.len().saturating_sub(1).max(1); // 5 + let max_match_span = needle.len() + 4; // 10 + + // Helper: check if a match would pass our post-filters + let passes = |n: &str, h: &str| -> bool { + let Some(mut mi) = neo_frizbee::match_list_indices(n, &[h], &config) + .into_iter() + .next() + else { + return false; + }; + // upstream returns indices in reverse order, sort ascending + mi.indices.sort_unstable(); + if mi.indices.len() < min_matched { + return false; + } + if let (Some(&first), Some(&last)) = (mi.indices.first(), mi.indices.last()) { + let span = last - first + 1; + if span > max_match_span { + return false; + } + let density = (mi.indices.len() * 100) / span; + if density < 70 { + return false; + } + } + true + }; + + // Exact match: must pass + assert!(passes("schema", "schema")); + // Exact in longer line: must pass + assert!(passes("schema", " schema: String,")); + // In identifier: must pass + assert!(passes("schema", "pub fn validate_schema() {}")); + // Transposition: must pass + assert!(passes("shcema", "schema")); + // Partial "ema" only line: must NOT pass + assert!(!passes("schema", "it has ema in it")); + // Completely unrelated: must NOT pass + assert!(!passes("schema", "hello world foo bar")); +} + +#[test] +fn test_multi_grep_search() { + use crate::file_picker::{FilePicker, FilePickerOptions}; + use std::io::Write; + + let dir = tempfile::tempdir().unwrap(); + + // File 1: has "GrepMode" and "GrepMatch" + { + let mut f = std::fs::File::create(dir.path().join("grep.rs")).unwrap(); + writeln!(f, "pub enum GrepMode {{").unwrap(); + writeln!(f, " PlainText,").unwrap(); + writeln!(f, " Regex,").unwrap(); + writeln!(f, "}}").unwrap(); + writeln!(f, "pub struct GrepMatch {{").unwrap(); + writeln!(f, " pub line_number: u64,").unwrap(); + writeln!(f, "}}").unwrap(); + } + + // File 2: has "PlainTextMatcher" only + { + let mut f = std::fs::File::create(dir.path().join("matcher.rs")).unwrap(); + writeln!(f, "struct PlainTextMatcher {{").unwrap(); + writeln!(f, " needle: Vec,").unwrap(); + writeln!(f, "}}").unwrap(); + } + + // File 3: no matches + { + let mut f = std::fs::File::create(dir.path().join("other.rs")).unwrap(); + writeln!(f, "fn main() {{").unwrap(); + writeln!(f, " println!(\"hello\");").unwrap(); + writeln!(f, "}}").unwrap(); + } + + let mut picker = FilePicker::new(FilePickerOptions { + base_path: dir.path().to_str().unwrap().into(), + watch: false, + ..Default::default() + }) + .unwrap(); + picker.collect_files().unwrap(); + + let files = picker.get_files(); + let arena = picker.arena_base_ptr(); + + let options = super::GrepSearchOptions { + max_file_size: MAX_FFFILE_SIZE, + max_matches_per_file: 0, + smart_case: true, + file_offset: 0, + page_limit: 100, + mode: super::GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let no_cancel = AtomicBool::new(false); + + // Test with 3 patterns + let result = super::multi_grep_search( + files, + &["GrepMode", "GrepMatch", "PlainTextMatcher"], + &[], + &options, + picker.cache_budget(), + None, + None, + &no_cancel, + dir.path(), + arena, + arena, + ); + + assert!( + result.matches.len() >= 3, + "Expected at least 3 matches, got {}", + result.matches.len() + ); + + let has_grep_mode = result + .matches + .iter() + .any(|m| m.line_content.contains("GrepMode")); + let has_grep_match = result + .matches + .iter() + .any(|m| m.line_content.contains("GrepMatch")); + let has_plain_text_matcher = result + .matches + .iter() + .any(|m| m.line_content.contains("PlainTextMatcher")); + + assert!(has_grep_mode, "Should find GrepMode"); + assert!(has_grep_match, "Should find GrepMatch"); + assert!(has_plain_text_matcher, "Should find PlainTextMatcher"); + + assert_eq!(result.files.len(), 2, "Should match exactly 2 files"); + + // Test with single pattern + let result2 = super::multi_grep_search( + files, + &["PlainTextMatcher"], + &[], + &options, + picker.cache_budget(), + None, + None, + &no_cancel, + dir.path(), + arena, + arena, + ); + assert_eq!( + result2.matches.len(), + 1, + "Single pattern should find 1 match" + ); + + // Test with empty patterns + let result3 = super::multi_grep_search( + files, + &[], + &[], + &options, + picker.cache_budget(), + None, + None, + &no_cancel, + dir.path(), + arena, + arena, + ); + assert_eq!( + result3.matches.len(), + 0, + "Empty patterns should find nothing" + ); +} + +/// Regression test for issue #407: Live grep returns duplicate results +/// when the bigram candidate bitset has trailing bits set beyond +/// `base_file_count`. The bitset is rounded up to a multiple of 64 bits +/// so any trailing bit that happens to be set (e.g. from overlay data) +/// would previously map to an overflow file index, which was then also +/// unconditionally appended by the overflow loop, producing duplicates. +#[test] +fn test_grep_no_duplicates_with_overflow_trailing_bits() { + let dir = tempfile::tempdir().unwrap(); + // Match the picker's internal dunce-canonicalize so paths passed to + // on_create_or_modify resolve back to the same base_path on Windows. + let base = crate::path_utils::canonicalize(dir.path()).unwrap(); + + // Five base files: only three contain the pattern "unicorn". + // We need some files WITHOUT the pattern so the bigrams for + // "unicorn" aren't treated as ubiquitous (≥90% of files) and + // dropped from the index during compress(). + let base_contents: &[(&str, &str)] = &[ + ("a.txt", "hello unicorn world"), + ("b.txt", "another unicorn line"), + ("c.txt", "one more unicorn here"), + ("d.txt", "nothing special in here"), + ("e.txt", "just some random content"), + ]; + for (name, content) in base_contents { + let mut f = std::fs::File::create(base.join(name)).unwrap(); + writeln!(f, "{}", content).unwrap(); + } + + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_str().unwrap().into(), + watch: false, + ..Default::default() + }) + .unwrap(); + picker.collect_files().unwrap(); + assert_eq!(picker.get_files().len(), 5); + + // Manually build a bigram index over the 5 base files. + let base_count = 5usize; + let consec_builder = BigramIndexBuilder::new(base_count); + let skip_builder = BigramIndexBuilder::new(base_count); + for (i, (_, content)) in base_contents.iter().enumerate() { + consec_builder.add_file_content(&skip_builder, i, content.as_bytes()); + } + let mut index = consec_builder.compress(Some(0)); + index.set_skip_index(skip_builder.compress(Some(0))); + picker.set_bigram_index(index); + + // Add three overflow files (new after the bigram index was built), + // all containing "unicorn". + for name in ["f.txt", "g.txt", "h.txt"] { + let path = base.join(name); + let mut f = std::fs::File::create(&path).unwrap(); + writeln!(f, "overflow unicorn entry").unwrap(); + drop(f); + picker.handle_create_or_modify(&path); + } + assert_eq!(picker.get_files().len(), 8); + + // Inject a trailing bit into the overlay at a file index that + // corresponds to an overflow file (i.e. >= base_file_count=5 but + // < bitset_word_size=64). Without the fix, the bigram-candidate + // merge would set this bit in the bitset, and the bitset loop would + // push files[6] while the overflow loop also appends files[5..] + // which includes files[6], producing a duplicate. + let overflow_rel = "g.txt"; // middle overflow file + let overflow_abs = picker + .get_files() + .iter() + .position(|f| f.relative_path(&picker) == overflow_rel) + .expect("overflow file should be present"); + assert!(overflow_abs >= base_count); + assert!( + overflow_abs < 64, + "index must fit in the single bitset word" + ); + + if let Some(overlay) = picker.bigram_overlay() { + overlay + .write() + .modify_file(overflow_abs, b"overflow unicorn entry"); + } + + // Run a grep for "unicorn": six files match + // (a, b, c in base + f, g, h in overflow). + let query = super::parse_grep_query("unicorn"); + let options = super::GrepSearchOptions { + max_file_size: MAX_FFFILE_SIZE, + max_matches_per_file: 0, + smart_case: true, + file_offset: 0, + page_limit: 100, + mode: super::GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: Some(std::sync::Arc::new(AtomicBool::new(false))), + }; + let result = picker.grep(&query, &options); + + // Collect the matched relative paths via the returned files list. + let mut paths: Vec = result + .files + .iter() + .map(|f| f.relative_path(&picker)) + .collect(); + paths.sort(); + + // Every file (base + overflow) should match exactly once. + let mut dedup = paths.clone(); + dedup.dedup(); + assert_eq!( + dedup, paths, + "grep must not return duplicate results (issue #407): {:?}", + paths + ); + assert_eq!( + paths, + vec!["a.txt", "b.txt", "c.txt", "f.txt", "g.txt", "h.txt"], + ); + + // And the match count must equal the number of files (one line per + // file). A duplicate entry in files_to_search would double-count + // matches for the duplicated file. + assert_eq!( + result.matches.len(), + 6, + "expected exactly one match per file, got {}", + result.matches.len() + ); +} diff --git a/crates/fff-core/src/grep/mod.rs b/crates/fff-core/src/grep/mod.rs new file mode 100644 index 0000000..eefe52b --- /dev/null +++ b/crates/fff-core/src/grep/mod.rs @@ -0,0 +1,16 @@ +mod fuzzy_grep; + +mod utils; +pub use utils::*; // contains some of the generally available functions and types + +#[allow(clippy::module_inception)] +mod grep; +pub use grep::*; + +#[cfg(feature = "definitions")] +mod classify; +#[cfg(feature = "definitions")] +pub use classify::*; + +#[cfg(test)] +mod grep_tests; diff --git a/crates/fff-core/src/grep/utils.rs b/crates/fff-core/src/grep/utils.rs new file mode 100644 index 0000000..37eb2b2 --- /dev/null +++ b/crates/fff-core/src/grep/utils.rs @@ -0,0 +1,122 @@ +use super::grep::{GrepMatch, GrepSearchOptions}; +use crate::types::FileItem; + +#[inline] +pub(crate) fn strip_line_terminators(bytes: &[u8]) -> &[u8] { + let mut len = bytes.len(); + while len > 0 && matches!(bytes[len - 1], b'\n' | b'\r') { + len -= 1; + } + &bytes[..len] +} + +/// Result of a grep search with a list of matches, list of matched files, and metadata. +#[derive(Debug, Clone, Default)] +pub struct GrepResult<'a> { + pub matches: Vec, + /// Deduplicated file references for the returned matches. + pub files: Vec<&'a FileItem>, + /// Number of files actually searched in this call. + pub total_files_searched: usize, + /// Total number of indexed files (before filtering). + pub total_files: usize, + /// Total number of searchable files (after filtering out binary, too-large, etc.). + pub filtered_file_count: usize, + /// Number of files that contained at least one match. + pub files_with_matches: usize, + /// The file offset to pass for the next page. `0` if there are no more files. + /// Callers should store this and pass it as `file_offset` in the next call. + pub next_file_offset: usize, + /// When regex mode fails to compile the pattern, the search falls back to + /// literal matching and this field contains the compilation error message. + /// The UI can display this to inform the user their regex was invalid. + pub regex_fallback_error: Option, +} + +impl<'a> GrepResult<'a> { + /// Empty result carrying only the file counts (empty query / prefilter miss) + pub(crate) fn empty(total_files: usize, filtered_file_count: usize) -> Self { + Self { + total_files, + filtered_file_count, + ..Default::default() + } + } + + pub(crate) fn collect( + per_file_results: Vec<(usize, &'a FileItem, Vec)>, + files_to_search_len: usize, + options: &GrepSearchOptions, + total_files: usize, + filtered_file_count: usize, + budget_exceeded: bool, + ) -> Self { + let page_limit = options.page_limit; + + // Each match stores a `file_index` pointing into `result_files` so that + // consumers (FFI JSON, Lua) can look up file metadata without duplicating + // it across every match from the same file + let mut result_files: Vec<&'a FileItem> = Vec::new(); + let mut all_matches: Vec = Vec::new(); + // files_consumed tracks how far into files_to_search we have advanced, + // counting every file whose results were emitted (with or without matches). + // We use the batch_idx of the last consumed file + 1, which is correct + // because per_file_results only contains files that had matches, and + // files between them that had no matches were still searched and can be + // safely skipped on the next page + let mut files_consumed: usize = 0; + + for (batch_idx, file, file_matches) in per_file_results { + // batch_idx is the 0-based position in files_to_search. + // Advance files_consumed to include this file and all no-match files before it. + files_consumed = batch_idx + 1; + + let file_result_idx = result_files.len(); + result_files.push(file); + + for mut m in file_matches { + m.file_index = file_result_idx; + if options.trim_whitespace { + m.trim_leading_whitespace(); + } + all_matches.push(m); + } + + // page_limit is a soft cap: we always finish the current file before + // stopping, so no matches are dropped. A page may return up to + // page_limit + max_matches_per_file - 1 matches in the worst case + if all_matches.len() >= page_limit { + break; + } + } + + // If no file had any match, we searched the entire slice. + if result_files.is_empty() { + files_consumed = files_to_search_len; + } + + let has_more = budget_exceeded + || (all_matches.len() >= page_limit && files_consumed < files_to_search_len); + + let next_file_offset = if has_more { + options.file_offset + files_consumed + } else { + 0 + }; + + Self { + matches: all_matches, + files_with_matches: result_files.len(), + files: result_files, + total_files_searched: files_consumed, + total_files, + filtered_file_count, + next_file_offset, + regex_fallback_error: None, + } + } +} + +pub fn has_regex_metacharacters(text: &str) -> bool { + regex::escape(text) != text +} diff --git a/crates/fff-core/src/ignore.rs b/crates/fff-core/src/ignore.rs new file mode 100644 index 0000000..ac29f52 --- /dev/null +++ b/crates/fff-core/src/ignore.rs @@ -0,0 +1,68 @@ +use std::path::Path; + +/// Directories excluded when walking a non-git root. Entries are `cfg`-gated +/// so a single iteration covers standard + platform-specific overrides. +pub(crate) const IGNORED_DIRS: &[&str] = &[ + "node_modules", + "__pycache__", + "venv", + ".venv", + // Rust (glob-only patterns for non_git_repo_overrides; is_non_code_directory + // matches the "target" component separately). + "target/debug", + "target/release", + "target/rust-analyzer", + "target/criterion", + #[cfg(target_os = "macos")] + "Library/Application Support", + #[cfg(target_os = "macos")] + "Library/Caches", + // App-group sandbox storage — used by iMessage, Photos, Notes, Calendar, + // Electron apps, etc. for SQLite-WAL, LevelDB, protobuf files. These are + // almost entirely extension-less binary files (~80k on a typical $HOME) + // that never need to appear in a fuzzy or grep search. + #[cfg(target_os = "macos")] + "Library/Group Containers", + #[cfg(target_os = "macos")] + "Library/Containers", + #[cfg(target_os = "windows")] + "bin/Debug", + #[cfg(target_os = "windows")] + "bin/Release", + #[cfg(target_os = "windows")] + "Program Files", + #[cfg(target_os = "windows")] + "Program Files (x86)", + #[cfg(target_os = "windows")] + "AppData/Local", + #[cfg(target_os = "windows")] + "AppData/Roaming", +]; + +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option { + use ignore::overrides::OverrideBuilder; + + let mut builder = OverrideBuilder::new(base_path); + for dir in IGNORED_DIRS { + let pattern = format!("!**/{dir}/"); + if let Err(e) = builder.add(&pattern) { + tracing::warn!("failed to add ignore pattern {pattern}: {e}"); + } + } + + builder.build().ok() +} + +pub(crate) fn is_non_code_directory(path: &Path) -> bool { + let path_str = path.as_os_str().to_str().unwrap_or(""); + IGNORED_DIRS.iter().any(|&dir| { + #[cfg(target_os = "windows")] + let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR); + #[cfg(target_os = "windows")] + return path_str.contains(dir.as_str()); + + #[cfg(not(target_os = "windows"))] + path_str.contains(dir) + }) +} diff --git a/crates/fff-core/src/lib.rs b/crates/fff-core/src/lib.rs new file mode 100644 index 0000000..434fcd8 --- /dev/null +++ b/crates/fff-core/src/lib.rs @@ -0,0 +1,161 @@ +//! # FFF Search — High-performance file finder core +//! +//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim). +//! It includes filesystem indexing with real-time watching, fuzzy matching powered +//! by [frizbee](https://docs.rs/neo_frizbee), frecency scoring backed by LMDB, +//! and multi-mode grep search. +//! +//! > [!Important performance information] +//! > For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine. +//! +//! ## Architecture +//! +//! - [`file_picker::FilePicker`] — Main entry point. Indexes a directory tree in a +//! background thread, maintains a sorted file list, watches the filesystem for +//! changes, and performs fuzzy search with frecency-weighted scoring. +//! - [`frecency::FrecencyTracker`] — LMDB-backed database that tracks file access +//! and modification patterns for intelligent result ranking. +//! - [`query_tracker::QueryTracker`] — Tracks search query history and provides +//! "combo-boost" scoring for repeatedly matched files. +//! - [`grep`] — Live grep search supporting regex, plain-text, and fuzzy modes +//! with optional constraint filtering. +//! - [`git`] — Git status caching and repository detection. +//! +//! ## Shared State +//! +//! [`SharedFilePicker`], [`SharedFrecency`], and [`SharedQueryTracker`] are +//! newtype wrappers around `Arc>>` for thread-safe shared +//! access. They provide `read()` / `write()` methods with built-in error +//! conversion and convenience helpers like `wait_for_scan()`. +//! +//! ## Quick Start +//! +//! ``` +//! use fff_search::file_picker::FilePicker; +//! use fff_search::frecency::FrecencyTracker; +//! use fff_search::query_tracker::QueryTracker; +//! use fff_search::{ +//! FFFMode, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser, +//! SharedFrecency, SharedFilePicker, SharedQueryTracker, +//! }; +//! +//! let shared_picker = SharedFilePicker::default(); +//! let shared_frecency = SharedFrecency::default(); +//! let shared_query_tracker = SharedQueryTracker::default(); +//! +//! let tmp = std::env::temp_dir().join("fff-doctest"); +//! std::fs::create_dir_all(&tmp).unwrap(); +//! +//! // 1. Optionally initialize frecency and query tracker databases +//! let frecency = FrecencyTracker::open(tmp.join("frecency"))?; +//! shared_frecency.init(frecency)?; +//! +//! let query_tracker = QueryTracker::open(tmp.join("queries"))?; +//! shared_query_tracker.init(query_tracker)?; +//! +//! // 2. Init the file picker (spawns background scan + watcher) +//! FilePicker::new_with_shared_state( +//! shared_picker.clone(), +//! shared_frecency.clone(), +//! FilePickerOptions { +//! base_path: ".".into(), +//! mode: FFFMode::Ai, +//! ..Default::default() +//! }, +//! )?; +//! +//! // 3. Wait for scan +//! shared_picker.wait_for_scan(std::time::Duration::from_secs(10)); +//! +//! // 4. Search: lock the picker and query tracker +//! let picker_guard = shared_picker.read()?; +//! let picker = picker_guard.as_ref().unwrap(); +//! let qt_guard = shared_query_tracker.read()?; +//! +//! // 5. Parse the query and perform fuzzy search +//! let parser = QueryParser::default(); +//! let query = parser.parse("lib.rs"); +//! +//! let results = picker.fuzzy_search( +//! &query, +//! qt_guard.as_ref(), +//! FuzzySearchOptions { +//! max_threads: 0, +//! current_file: None, +//! pagination: PaginationArgs { offset: 0, limit: 50 }, +//! ..Default::default() +//! }, +//! ); +//! +//! assert!(results.total_matched > 0); +//! assert!(results.items.first().unwrap().relative_path(picker).ends_with("lib.rs")); +//! +//! let _ = std::fs::remove_dir_all(&tmp); +//! # Ok::<(), Box>(()) +//! ``` + +#[cfg(not(any(feature = "ripgrep", feature = "zlob")))] +compile_error!( + "fff-search requires either the `ripgrep` (default) or `zlob` feature. \ + Enable one, e.g. `--features ripgrep` or `--features zlob`." +); + +/// Primary entry points with thread-safe [`SharedFilePicker`](shared::FilePicker) instance +pub mod shared; +pub use shared::*; + +/// Core file picker single thread: filesystem indexing, background watching, and fuzzy search. +/// See [`FilePicker`](file_picker::FilePicker) for the main entry point. +pub mod file_picker; +pub use file_picker::*; + +/// Database-backed persistence: frecency, query history, LMDB plumbing. +pub mod dbs; +pub use dbs::*; + +/// Git status caching and repository detection utilities. +pub mod git; + +/// Live grep search with regex, plain-text, and fuzzy matching modes. +pub mod grep; +pub use grep::*; + +/// Tracing/logging initialization and panic hook setup. +pub mod log; + +/// Various path utils might be handy for you to work with fff paths +pub mod path_utils; + +/// Core data types shared across the crate. +pub mod types; +pub use types::*; + +pub mod constants; + +// ================================== +// these are public only for benchmarks, no backward compatibility guaranteed +#[doc(hidden)] +pub mod bigram_filter; +#[doc(hidden)] +pub mod simd_string_utils; +// ================================== + +mod background_watcher; +mod constraints; +mod error; +mod git_status_worker; +mod ignore; +mod scan; +mod score; +mod sort_buffer; + +pub(crate) mod bigram_query; +pub(crate) mod parallelism; +pub(crate) mod simd_path; +pub(crate) mod stable_vec; +pub(crate) mod walk; + +// fff error +pub use error::{Error, Result}; + +pub use fff_query_parser::*; diff --git a/crates/fff-core/src/log.rs b/crates/fff-core/src/log.rs new file mode 100644 index 0000000..1e58648 --- /dev/null +++ b/crates/fff-core/src/log.rs @@ -0,0 +1,281 @@ +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use tracing_appender::non_blocking; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{EnvFilter, fmt, prelude::*}; + +// Set once on first init_tracing; doubles as the init-once gate. +static LOG_FILE_PATH: OnceLock = OnceLock::new(); +static CRASH_HOOKS: OnceLock<()> = OnceLock::new(); + +fn write_crash_report(header: &str, body: &str) { + let msg = format!( + "\n=== CRASH (this might NOT BE fff related) {} ===\n{}\n=== CRASH END {} ===\n", + header, body, header + ); + let _ = std::io::Write::write_all(&mut std::io::stderr(), msg.as_bytes()); + if let Some(path) = LOG_FILE_PATH.get() { + let _ = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .and_then(|mut f| std::io::Write::write_all(&mut f, msg.as_bytes())); + } +} + +// SIGSEGV handler writes a banner to a pre-opened fd (open(2) inside a signal +// handler is unsafe due to path-resolution allocs). Unix only. +#[cfg(unix)] +mod sigsegv { + use std::os::fd::IntoRawFd; + use std::path::Path; + use std::sync::atomic::{AtomicI32, Ordering}; + + static LOG_FD: AtomicI32 = AtomicI32::new(-1); + + // Must `create(true)` — this runs before init_tracing opens/creates the + // writer file, so an append-only open on a non-existent path silently + // fails, LOG_FD stays -1, and the SIGSEGV banner never reaches the log. + pub fn set_log_fd(path: &Path) { + if let Ok(file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let prev = LOG_FD.swap(file.into_raw_fd(), Ordering::Relaxed); + if prev >= 0 { + unsafe { libc::close(prev) }; + } + } + } + + // Body must be async-signal-safe: write(2), atomic load, signal(2). Nothing else. + fn handler(_info: &libc::siginfo_t) { + const BANNER: &[u8] = b"\n=== CRASH SIGSEGV (fff) ===\n\ + fff.nvim's rust extension hit a segfault and is about to die.\n\ + Please file the bug at https://github.com/dmtrKovalenko/fff/issues with this banner attached.\n\ + === CRASH END SIGSEGV ===\n"; + unsafe { + libc::write(2, BANNER.as_ptr().cast(), BANNER.len()); + let log_fd = LOG_FD.load(Ordering::Relaxed); + if log_fd >= 0 { + libc::write(log_fd, BANNER.as_ptr().cast(), BANNER.len()); + } + // Reset to default so handler return → kernel kills us instead of + // re-running the faulting instruction in an infinite loop. + libc::signal(libc::SIGSEGV, libc::SIG_DFL); + } + } + + pub fn install() { + // signal-hook-registry chains to LuaJIT's prior handler automatically. + unsafe { + let _ = signal_hook_registry::register_unchecked(libc::SIGSEGV, handler); + } + } +} + +#[cfg(not(unix))] +mod sigsegv { + use std::path::Path; + pub fn set_log_fd(_path: &Path) {} + pub fn install() {} +} + +pub fn install_panic_hook() { + CRASH_HOOKS.get_or_init(install_crash_hooks); +} + +fn install_crash_hooks() { + let default_panic = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = panic_info.payload().downcast_ref::() { + s.clone() + } else { + "Unknown panic payload".to_string() + }; + + let location = panic_info + .location() + .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column())) + .unwrap_or_else(|| "unknown location".to_string()); + + tracing::error!( + panic.message = %message, + panic.location = %location, + "PANIC occurred in FFF" + ); + + write_crash_report( + "RUST PANIC", + &format!("Message: {}\nLocation: {}", message, location), + ); + default_panic(panic_info); + })); + + sigsegv::install(); +} + +/// Parse a log level string into a `tracing::Level`. +pub fn parse_log_level(level: Option<&str>) -> tracing::Level { + match level.as_ref().map(|s| s.trim().to_lowercase()).as_deref() { + Some("trace") => tracing::Level::TRACE, + Some("debug") => tracing::Level::DEBUG, + Some("info") => tracing::Level::INFO, + Some("warn") => tracing::Level::WARN, + Some("error") => tracing::Level::ERROR, + _ => tracing::Level::INFO, + } +} + +/// Default retention: how many prior nvim sessions' log files to keep. +const DEFAULT_RETAIN_RUNS: usize = 20; + +pub fn generate_trace_id() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static TRACE_COUNTER: AtomicU64 = AtomicU64::new(0); + + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + + let pid = std::process::id() as u64; + let counter = TRACE_COUNTER.fetch_add(1, Ordering::Relaxed); + + // very simple hash functions helps to distinguish trace ids visually + let id = nanos ^ (pid.wrapping_mul(0x9E37_79B9_7F4A_7C15)) ^ (counter << 32); + format!("{:016x}", id) +} + +pub fn trace_span(trace_id: &str, label: &'static str) -> tracing::Span { + tracing::info_span!("fff.trace", trace_id = trace_id, label = label) +} + +fn unix_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn session_path_from_hint(hint: &Path) -> PathBuf { + let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff"); + let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log"); + let parent = hint.parent().unwrap_or_else(|| Path::new(".")); + parent.join(format!( + "{stem}+{ts}+{pid}.{ext}", + ts = unix_secs(), + pid = std::process::id(), + )) +} + +fn rotate_logs(dir: &Path, stem: &str, ext: &str, retain_runs: usize) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let prefix = format!("{stem}+"); + let suffix = format!(".{ext}"); + + let mut files: Vec<(std::time::SystemTime, PathBuf)> = entries + .filter_map(|res| { + let entry = res.ok()?; + let name = entry.file_name(); + let name = name.to_str()?; + if !name.starts_with(&prefix) || !name.ends_with(&suffix) { + return None; + } + + let mtime = entry.metadata().ok()?.modified().ok()?; + Some((mtime, entry.path())) + }) + .collect(); + + if files.len() <= retain_runs { + return; + } + // Newest first, then drop everything past retain_runs. + files.sort_by_key(|(mtime, _)| std::cmp::Reverse(*mtime)); + for (_, path) in files.into_iter().skip(retain_runs) { + let _ = std::fs::remove_file(path); + } +} + +/// `log_file_path` is a path-shape hint. Each call writes a unique sibling +/// `++.` so concurrent processes never collide. +/// Returns the absolute path of the session file. +pub fn init_tracing( + log_file_path: &str, + log_level: Option<&str>, + retain_runs: Option, +) -> Result { + let hint = Path::new(log_file_path); + let session_dir = hint + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + std::fs::create_dir_all(&session_dir)?; + + let session_path = session_path_from_hint(hint); + + // First init wins; repeat callers no-op and return the original path. + if LOG_FILE_PATH.set(session_path.clone()).is_err() { + return Ok(LOG_FILE_PATH + .get() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default()); + } + + sigsegv::set_log_fd(&session_path); + install_panic_hook(); + + let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff"); + let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log"); + rotate_logs( + &session_dir, + stem, + ext, + retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS), + ); + + let writer_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&session_path)?; + + // we intinionally leark the guard we don't ever want to stop logging + let (non_blocking_appender, guard) = non_blocking(writer_file); + Box::leak(Box::new(guard)); + + let subscriber = tracing_subscriber::registry() + .with( + fmt::layer() + .with_writer(non_blocking_appender) + .with_target(true) + .with_thread_ids(false) + .with_thread_names(true) + .with_ansi(false) + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE), + ) + .with( + EnvFilter::builder() + .with_default_directive(parse_log_level(log_level).into()) + .from_env_lossy(), + ); + + if let Err(e) = tracing::subscriber::set_global_default(subscriber) { + eprintln!("Failed to set tracing subscriber: {}", e); + } else { + tracing::info!( + "FFF tracing initialized: {} (pid={}, retain_runs={})", + session_path.display(), + std::process::id(), + retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS), + ); + } + + Ok(session_path.to_string_lossy().into_owned()) +} diff --git a/crates/fff-core/src/parallelism.rs b/crates/fff-core/src/parallelism.rs new file mode 100644 index 0000000..7f6c3bb --- /dev/null +++ b/crates/fff-core/src/parallelism.rs @@ -0,0 +1,84 @@ +//! Dedicated rayon pools. The global pool spans every logical core, which +//! oversubscribes asymmetric chips (Apple P+E): E-cores are ~2× slower and +//! `open()` contends on a per-VFS lock past P-core count, so a larger pool is +//! slower on file-heavy work. + +use std::sync::LazyLock; + +/// Dedicated thread pool for background work (scan, warmup, bigram build). +pub static BACKGROUND_THREAD_POOL: LazyLock = LazyLock::new(|| { + let total = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(4); + + // Background work is mostly syscall-bound; halving parallelism leaves + // cores for search/UI at negligible throughput cost. + let bg_threads = (total / 2).max(2); + rayon::ThreadPoolBuilder::new() + .num_threads(bg_threads) + .thread_name(|i| format!("fff-bg-{i}")) + .start_handler(|_| { + // QoS pin keeps workers on P-cores; the kernel otherwise drifts + // them to ~2× slower E-cores. + #[cfg(target_os = "macos")] + unsafe { + let _ = libc::pthread_set_qos_class_self_np( + libc::qos_class_t::QOS_CLASS_USER_INITIATED, + 0, + ); + } + }) + .build() + .expect("failed to create background rayon pool") +}); + +/// Physical performance-core count via sysctl, falling back to logical cores. +/// On a 12P+4E M4 Max, grep runs 16t=6.2s vs 13t=4.9s — fewer threads win. +#[cfg(target_os = "macos")] +fn performance_core_count() -> usize { + let mut count: libc::c_int = 0; + let mut size = std::mem::size_of::(); + let name = c"hw.perflevel0.physicalcpu"; + let ok = unsafe { + libc::sysctlbyname( + name.as_ptr(), + &mut count as *mut _ as *mut libc::c_void, + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + if ok == 0 && count > 0 { + count as usize + } else { + std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(4) + } +} + +/// Pool for grep content search: P-core sized and QoS-pinned on macOS, full +/// parallelism elsewhere. Avoids E-core drag and VFS-lock contention. +pub static SEARCH_THREAD_POOL: LazyLock = LazyLock::new(|| { + #[cfg(target_os = "macos")] + let threads = performance_core_count(); + #[cfg(not(target_os = "macos"))] + let threads = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(4); + + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .thread_name(|i| format!("fff-search-{i}")) + .start_handler(|_| { + #[cfg(target_os = "macos")] + unsafe { + let _ = libc::pthread_set_qos_class_self_np( + libc::qos_class_t::QOS_CLASS_USER_INITIATED, + 0, + ); + } + }) + .build() + .expect("failed to create search rayon pool") +}); diff --git a/crates/fff-core/src/path_utils.rs b/crates/fff-core/src/path_utils.rs new file mode 100644 index 0000000..a63d655 --- /dev/null +++ b/crates/fff-core/src/path_utils.rs @@ -0,0 +1,211 @@ +use std::path::{Path, PathBuf}; + +#[cfg(windows)] +pub fn canonicalize(path: impl AsRef) -> std::io::Result { + dunce::canonicalize(path) +} + +#[cfg(not(windows))] +pub fn canonicalize(path: impl AsRef) -> std::io::Result { + std::fs::canonicalize(path) +} + +/// The index stores relative paths with `/` on every platform. These helpers +/// convert between that canonical form and the OS-native separator, and are +/// no-ops on non-Windows where `/` is already native. + +/// Fold a relative path to the canonical `/` form (no-op off Windows). +#[cfg(windows)] +pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> { + if rel.contains('\\') { + std::borrow::Cow::Owned(rel.replace('\\', "/")) + } else { + std::borrow::Cow::Borrowed(rel) + } +} + +#[cfg(not(windows))] +#[inline] +pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> { + std::borrow::Cow::Borrowed(rel) +} + +/// Rewrite canonical `/` bytes to the OS-native separator in place (no-op off +/// Windows). Used at OS/state boundaries (absolute-path reconstruction). +#[cfg(windows)] +#[inline] +pub fn nativize_slashes_in_place(bytes: &mut [u8]) { + for b in bytes { + if *b == b'/' { + *b = b'\\'; + } + } +} + +#[cfg(not(windows))] +#[inline] +pub fn nativize_slashes_in_place(_bytes: &mut [u8]) {} + +/// Git requires a normalized forward-slashed paths on windows +#[cfg(windows)] +pub fn normalize(path: PathBuf) -> PathBuf { + let as_str = path.to_string_lossy(); + let with_backslashes: String = as_str.replace('/', "\\"); + let buf = PathBuf::from(with_backslashes); + dunce::canonicalize(&buf).unwrap_or(buf) +} + +#[cfg(not(windows))] +pub fn normalize(path: PathBuf) -> PathBuf { + path +} + +#[cfg(windows)] +pub fn expand_tilde(path: &str) -> PathBuf { + return PathBuf::from(path); +} + +#[cfg(not(windows))] +pub fn expand_tilde(path: &str) -> PathBuf { + if let Some(stripped) = path.strip_prefix("~/") + && let Some(home_dir) = dirs::home_dir() + { + return home_dir.join(stripped); + } + + PathBuf::from(path) +} + +/// Calculate distance penalty based on directory proximity. +/// Returns a negative penalty score based on how far the candidate is from the current file. +/// +/// `candidate_dir` is the directory portion of the candidate path (e.g. `"src/components/"`). +/// It may have a trailing `/` which is stripped internally. +/// +/// Zero-allocation: walks both directory part iterators in lockstep. +pub fn calculate_distance_penalty(current_file: Option<&str>, candidate_dir: &str) -> i32 { + let Some(current_path) = current_file else { + return 0; + }; + + let current_dir = Path::new(current_path).parent().unwrap_or(Path::new("")); + let candidate = Path::new(candidate_dir); + + if current_dir == candidate { + return 0; + } + + let mut current_parts = current_dir.components(); + let mut candidate_parts = candidate.components(); + + let mut common_len = 0usize; + let mut current_total = 0usize; + + loop { + match (current_parts.next(), candidate_parts.next()) { + (Some(a), Some(b)) => { + current_total += 1; + if a == b { + common_len += 1; + } else { + current_total += current_parts.count(); + break; + } + } + (Some(_), None) => { + current_total += 1 + current_parts.count(); + break; + } + (None, _) => { + break; + } + } + } + + let depth_from_common = current_total - common_len; + if depth_from_common == 0 { + return 0; + } + + (-(depth_from_common as i32)).max(-20) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[cfg(not(target_family = "windows"))] + fn test_calculate_distance_penalty() { + // candidate_dir is now just the directory portion (with or without trailing /) + assert_eq!(calculate_distance_penalty(None, "examples/user/test/"), 0); + // Same directory + assert_eq!( + calculate_distance_penalty(Some("examples/user/test/main.rs"), "examples/user/test/"), + 0 + ); + // + // One level apart + assert_eq!( + calculate_distance_penalty( + Some("examples/user/test/subdir/file.rs"), + "examples/user/test/" + ), + -1 + ); + // + // Different subdirectories (same parent) + assert_eq!( + calculate_distance_penalty( + Some("examples/user/test/dir1/file.rs"), + "examples/user/test/dir2/" + ), + -1 + ); + + assert_eq!( + calculate_distance_penalty( + Some("examples/audio-announce/src/lib/audio-announce.rs"), + "examples/audio-announce/src/" + ), + -1 + ); + + assert_eq!( + calculate_distance_penalty( + Some("examples/audio-announce/src/audio-announce.rs"), + "examples/pixel/src/" + ), + -2 + ); + + // Root level files (empty dir) + assert_eq!(calculate_distance_penalty(Some("main.rs"), ""), 0); + } + + #[test] + #[cfg(target_family = "windows")] + fn distance_penalty_works_on_windows() { + assert_eq!( + calculate_distance_penalty(None, "examples\\user\\test\\"), + 0 + ); + // Same directory + assert_eq!( + calculate_distance_penalty( + Some("examples\\user\\test\\main.rs"), + "examples\\user\\test\\" + ), + 0 + ); + // + // One level apart + assert_eq!( + calculate_distance_penalty( + Some("examples\\user\\test\\subdir\\file.rs"), + "examples\\user\\test\\" + ), + -1 + ); + } +} diff --git a/crates/fff-core/src/scan.rs b/crates/fff-core/src/scan.rs new file mode 100644 index 0000000..3fe6fa3 --- /dev/null +++ b/crates/fff-core/src/scan.rs @@ -0,0 +1,404 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use tracing::{error, info}; + +use crate::FileSync; +use crate::background_watcher::BackgroundWatcher; +use crate::bigram_filter::{build_bigram_index, sniff_binary_for_non_indexable}; +use crate::error::Error; +use crate::file_picker::FFFMode; +use crate::parallelism::BACKGROUND_THREAD_POOL; +use crate::shared::{SharedFilePicker, SharedFrecency}; +use crate::types::ContentCacheBudget; + +#[derive(Clone, Default)] +pub(crate) struct ScanSignals { + /// Set to `true` while any scan phase is running + pub(crate) scanning: Arc, + /// Set to `true` once the filesystem watcher has been installed + pub(crate) watcher_ready: Arc, + /// Indicates that that owning picker was requested to shut down + pub(crate) cancelled: Arc, + /// Used to resolve conflicts if multiple rescans were triggered in a queue + pub(crate) rescan_pending: Arc, + /// Set by `post_scan_snapshot`, cleared by `PostScanSnapshot::drop`. + /// DO NOT set or clear this manually — it is managed exclusively by the + /// PostScanSnapshot lifecycle. + pub(crate) post_scan_indexing_active: Arc, +} + +/// Which optional phases a scan should run. +#[derive(Clone, Copy, Default, Debug)] +pub(crate) struct ScanConfig { + pub(crate) warmup: bool, + pub(crate) content_indexing: bool, + pub(crate) watch: bool, + pub(crate) auto_cache_budget: bool, + pub(crate) install_watcher: bool, + pub(crate) follow_symlinks: bool, + pub(crate) enable_fs_root_scanning: bool, + pub(crate) enable_home_dir_scanning: bool, +} + +/// A fully-configured scan job ready to run on a background thread. +/// +/// Build with [`ScanJob::from_picker`] (reads all state from the +/// current `FilePicker`) or [`ScanJob::initial`] (for the bootstrap +/// scan, before the picker is published to `SharedPicker`). +pub(crate) struct ScanJob { + shared_picker: SharedFilePicker, + shared_frecency: SharedFrecency, + base_path: PathBuf, + mode: FFFMode, + signals: ScanSignals, + config: ScanConfig, + /// Walker-maintained counter backing `get_scan_progress` on the UI + /// side. Reset to 0 at scan start, incremented per-file by the + /// walker. Shared `Arc` so the UI polls the same atomic. + scanned_files_counter: Arc, + trace_span: tracing::Span, +} + +impl ScanJob { + pub fn new_rescan( + shared_picker: &SharedFilePicker, + shared_frecency: &SharedFrecency, + ) -> Result, Error> { + let guard = shared_picker.read()?; + let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?; + + if picker.is_scan_active() + || picker + .signals + .post_scan_indexing_active + .load(Ordering::Acquire) + { + return Ok(None); + } + + let mode = picker.mode(); + let signals = picker.scan_signals(); + let scanned_files_counter = picker.scanned_files_counter(); + let base_path = picker.base_path().to_path_buf(); + let trace_span = picker.trace_span(); + + let new_scan_config = ScanConfig { + warmup: picker.has_mmap_cache(), + content_indexing: picker.has_content_indexing(), + watch: picker.has_watcher(), + auto_cache_budget: !picker.has_explicit_cache_budget(), + install_watcher: false, // the watcher is independent of rescan, it is not restarting EVER + follow_symlinks: picker.follows_symlinks(), + enable_fs_root_scanning: picker.fs_root_scanning_enabled(), + enable_home_dir_scanning: picker.home_dir_scanning_enabled(), + }; + + drop(guard); // just a sanity check + + Ok(Some(Self { + mode, + signals, + base_path, + scanned_files_counter, + config: new_scan_config, + shared_picker: shared_picker.clone(), + shared_frecency: shared_frecency.clone(), + trace_span, + })) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_initial( + shared_picker: SharedFilePicker, + shared_frecency: SharedFrecency, + base_path: PathBuf, + mode: FFFMode, + signals: ScanSignals, + scanned_files_counter: Arc, + trace_span: tracing::Span, + config: ScanConfig, + ) -> Self { + Self { + shared_picker, + shared_frecency, + base_path, + mode, + signals, + scanned_files_counter, + config, + trace_span, + } + } + + /// Run the job on `BACKGROUND_THREAD_POOL`. Returns immediately. + /// + /// Routed through the pool — and not a fresh `std::thread::spawn` — so the + /// orchestrator inherits rayon's QoS pin (USER_INITIATED). Without that + /// pin, an interactive nvim's USER_INTERACTIVE main thread spawns a child + /// at lower QoS, the walker's Zig worker pool inherits the demotion, and + /// the kernel drifts those workers onto E-cores. On chromium that turns a + /// ~800 ms walk into ~3 s. + pub fn spawn(self) { + self.signals.scanning.store(true, Ordering::Release); + let span = self.trace_span.clone(); + BACKGROUND_THREAD_POOL.spawn(move || { + let _g = span.enter(); + self.run(); + }); + } + + fn run(self) { + let Self { + shared_picker, + shared_frecency, + base_path, + mode, + signals, + scanned_files_counter, + config, + trace_span: _, + } = self; + + let _scanning = ScanningGuard::new(&signals, config.install_watcher); + scanned_files_counter.store(0, Ordering::Relaxed); + + // 1. Walk the file system and collect the list of files + let git_workdir = FileSync::discover_git_workdir(&base_path); + let sync = match FileSync::walk_filesystem( + &base_path, + git_workdir.clone(), + &scanned_files_counter, + &shared_frecency, + mode, + config.follow_symlinks, + ) { + Ok(sync) => sync, + Err(e) => { + error!(?e, "scan walk failed"); + return; + } + }; + + // 2. Populate the file list + let git_status_worker; + if let Ok(mut guard) = shared_picker.write() + && let Some(picker) = guard.as_mut() + { + if signals.cancelled.load(Ordering::Acquire) { + info!("scan cancelled between walk and commit, discarding"); + return; + } + + let live_count = sync.live_count; + picker.commit_new_sync(sync); + git_status_worker = Arc::clone(&picker.git_status_worker); + + if config.auto_cache_budget && !picker.has_explicit_cache_budget() { + picker.set_cache_budget(ContentCacheBudget::new_for_repo(live_count)); + } + } else { + error!("failed to install scan results into picker"); + return; + } + + // Spawn the git status worker once. BUG PINNNING. If the user initiated git in the folder + // which is a real use case we need to have a way to start the git worker background thread dynamically + if git_workdir.is_some() && !signals.cancelled.load(Ordering::Acquire) { + git_status_worker.spawn_once(shared_picker.weaken(), shared_frecency.clone()); + git_status_worker.request_full_rescan(); // this runs anyway + } + + // BUG pinning: take the snapshot *before* the storing the scan=true, otherwise there is a tiny + // race window when there scanned is set to true, but `post_scan_indexing_active` flag is `false` + let snapshot = if !signals.cancelled.load(Ordering::Acquire) { + shared_picker.read().ok().and_then(|guard| { + guard + .as_ref() + .and_then(|picker| unsafe { picker.post_scan_snapshot() }) + }) + } else { + None + }; + + signals.scanning.store(false, Ordering::Relaxed); // file are searchable + + // in case we do a rescan, we have to resubscribe a watcher to the new set of directories + // all the already watched directories are not going to be resubscribed (this is internally deduped) + if !config.install_watcher && !signals.cancelled.load(Ordering::Acquire) { + rescubscribe_watcher_post_scan(&shared_picker); + } + + // 3. Runs post scna in parallel with git status collection + if !signals.cancelled.load(Ordering::Acquire) + && let Some(snap) = snapshot.as_ref() + { + Self::run_post_scan(&shared_picker, &signals, &config, snap); + } + + drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT + + // 5. Install filesystem watcher (initial scan only). + if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) { + let shared_picker: &SharedFilePicker = &shared_picker; + let shared_frecency: &SharedFrecency = &shared_frecency; + let base_path: &std::path::Path = &base_path; + + match BackgroundWatcher::new( + base_path.to_path_buf(), + git_workdir, + shared_picker.clone(), + shared_frecency.clone(), + mode, + config.enable_fs_root_scanning, + config.enable_home_dir_scanning, + git_status_worker, + tracing::Span::current(), + ) { + Ok(watcher) => { + if let Ok(mut guard) = shared_picker.write() + && let Some(picker) = guard.as_mut() + { + picker.background_watcher = Some(watcher); + } + } + Err(e) => error!(?e, "failed to initialize background watcher"), + }; + } + + // 6. Drain any rescan that arrived while we were busy. + // if user initiated a new rescan we had no way to cancel current post scan, so do it again + if !signals.cancelled.load(Ordering::Acquire) + && signals.rescan_pending.swap(false, Ordering::AcqRel) + { + match Self::new_rescan(&shared_picker, &shared_frecency) { + Ok(Some(follow_up)) => { + info!("Rescheduling deferred rescan after current scan finished"); + follow_up.spawn(); + } + Ok(None) => { + // this should be practically impossible because we do not have any + // queue, but if somehow a new rescan was triggered JUST IN THIS MOMENT + // just ignore it because the ongoing one is fresh enough + tracing::warn!("Post scan was re-triggered, ignoring"); + } + Err(e) => { + error!(?e, "Failed to reschedule deferred rescan"); + } + } + } + } + + /// THIS IS VERY VERY IMPORTANT THAT ANYTHING INSIDE THIS FUNCTION TO NOT READ ANYTHING CLEARABLE OUTSIDE + /// this is a very silly off lock implementation that actually matters, and that's why it is crafted + /// to never read anything from the picker, it can only WRITE information using single instructions + /// + /// Things that are safe and immutable - file list, indexes of files, paths, and signals. + #[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))] + fn run_post_scan( + shared_picker: &SharedFilePicker, + signals: &ScanSignals, + config: &ScanConfig, + unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot, + ) { + let Some(arena) = unsafe_snapshot + .arena // we are never touching overlays so this arena is always correct + .as_ref() + .map(|s| s.as_arena_ptr()) + else { + tracing::error!("Failed to run post scan: arena is invalid"); + return; + }; + + let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count]; + if signals.cancelled.load(Ordering::Acquire) { + return; + } + + if config.content_indexing { + let indexable_count = unsafe_snapshot.indexable_count.min(files.len()); + let (indexable_files, non_indexable_files) = files.split_at(indexable_count); + let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena); + + if let Ok(mut guard) = shared_picker.write() + && let Some(picker) = guard.as_mut() + { + picker.set_bigram_index(index); + } + + // Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large + // unknown-extension binaries slip past it and would otherwise be + // grep-able as text. Cheap header sniff catches those. + if !signals.cancelled.load(Ordering::Acquire) { + sniff_binary_for_non_indexable( + non_indexable_files, + &unsafe_snapshot.base_path, + arena, + &signals.cancelled, + ); + } + } else { + // this potentially a long running as we are not parallelizing it but it's okay + sniff_binary_for_non_indexable( + files, + &unsafe_snapshot.base_path, + arena, + &signals.cancelled, + ); + } + + // TODO Skipped as potentially unsafe - figure this out later + // if config.warmup && !signals.cancelled.load(Ordering::Acquire) { + // warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena); + // } + } +} + +/// RAII helper that flips the `scanning` signal on construction and +/// resets it on drop (so early-returns can't leave it stuck on `true`). +/// Also drives the `watcher_ready` signal on the initial-scan path. +struct ScanningGuard<'a> { + signals: &'a ScanSignals, + release_watcher_ready_on_drop: bool, +} + +impl<'a> ScanningGuard<'a> { + fn new(signals: &'a ScanSignals, release_watcher_ready_on_drop: bool) -> Self { + signals.scanning.store(true, Ordering::Relaxed); + Self { + signals, + release_watcher_ready_on_drop, + } + } +} + +impl Drop for ScanningGuard<'_> { + fn drop(&mut self) { + self.signals.scanning.store(false, Ordering::Relaxed); + if self.release_watcher_ready_on_drop { + self.signals.watcher_ready.store(true, Ordering::Release); + } + } +} + +/// If the scan encounters new directories created we have to add them to the watch list +/// this is fine because the watcher does deduplicate the entries and doesn't add a lot of +/// garbage notify watchers / fs events streams +#[tracing::instrument(skip_all)] +fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) { + let Ok(guard) = shared_picker.read() else { + return; + }; + let Some(picker) = guard.as_ref() else { + return; + }; + let Some(watcher) = picker.background_watcher.as_ref() else { + return; + }; + + picker.for_each_dir(|dir: &std::path::Path| { + watcher.request_watch_dir(dir.to_path_buf()); + std::ops::ControlFlow::Continue(()) + }); +} diff --git a/crates/fff-core/src/score.rs b/crates/fff-core/src/score.rs new file mode 100644 index 0000000..f757f48 --- /dev/null +++ b/crates/fff-core/src/score.rs @@ -0,0 +1,1604 @@ +use crate::{ + constraints::apply_constraints, + git::is_modified_status, + path_utils::calculate_distance_penalty, + simd_path::{ArenaPtr, MAX_PATH_CHUNKS}, + sort_buffer::{sort_by_key_with_buffer, sort_with_buffer}, + types::{DirItem, FileItem, Score, ScoringContext}, +}; +use fff_query_parser::FuzzyQuery; +use neo_frizbee::Scoring; +use rayon::prelude::*; +use std::{borrow::Cow, path::MAIN_SEPARATOR}; + +enum FileItems<'a> { + All(&'a [FileItem]), + Filtered(Vec<&'a FileItem>), +} + +impl<'a> FileItems<'a> { + #[inline] + fn index(&self, index: usize) -> &'a FileItem { + match self { + FileItems::All(s) => &s[index], + FileItems::Filtered(v) => v[index], + } + } +} + +/// Resolve a FileItem's chunked path into frizbee's pointer buffer. +/// Returns `Some((chunk_count, byte_len))` or `None` for deleted files. +#[inline] +fn resolve_file_chunks( + file: &FileItem, + arena: ArenaPtr, + buf: &mut [*const u8; MAX_PATH_CHUNKS], +) -> Option<(usize, u16)> { + if file.is_deleted() { + return None; + } + let ptrs = file.path.resolve_ptrs(arena, buf); + Some((ptrs.len(), file.path.byte_len)) +} + +#[inline] +fn match_fuzzy_parts( + fuzzy_parts: &[&str], + working_files: &FileItems<'_>, + options: &neo_frizbee::Config, + max_threads: usize, + arena: ArenaPtr, +) -> Vec { + let valid_parts: Vec<&str> = fuzzy_parts + .iter() + .copied() + .filter(|p| p.len() >= 2) + .collect(); + + if valid_parts.is_empty() { + tracing::debug!("match_fuzzy_parts: no valid parts after filtering, returning empty"); + return vec![]; + } + + let resolve = |file: &FileItem, + buf: &mut [*const u8; MAX_PATH_CHUNKS]| + -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) }; + + // because we reassemble the vec of reference we have to use a different type + // to narrow down the [&FileItem] which would be resolved by frizbee as && + let resolve_ref = |file: &&FileItem, + buf: &mut [*const u8; MAX_PATH_CHUNKS]| + -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) }; + + let first_part_matches = match working_files { + FileItems::All(files) => neo_frizbee::match_list_parallel_resolved( + valid_parts[0], + files, + &resolve, + options, + max_threads, + ), + FileItems::Filtered(files) => neo_frizbee::match_list_parallel_resolved( + valid_parts[0], + files.as_slice(), + &resolve_ref, + options, + max_threads, + ), + }; + + if valid_parts.len() == 1 { + return first_part_matches; + } + + let total_parts = valid_parts.len() as u32; + let mut matches = first_part_matches; + for part in valid_parts[1..].iter() { + let mut part_options = *options; + part_options.max_typos = options.max_typos.map(|t| t.min(part.len() as u16)); + + // Collect the subset of files that survived the previous round. + let subset: Vec<&FileItem> = matches + .iter() + .map(|m| working_files.index(m.index as usize)) + .collect(); + + let sub_matches = neo_frizbee::match_list_parallel_resolved( + part, + subset.as_slice(), + &resolve_ref, + &part_options, + max_threads, + ); + + if sub_matches.is_empty() { + return vec![]; // break early! + } + + // Map sub_matches back to original indices, average scores across all parts. + matches = sub_matches + .into_iter() + .map(|sm| { + let prev = &matches[sm.index as usize]; + let sum = (prev.score as u32).saturating_add(sm.score as u32); + let avg = sum / total_parts; + + neo_frizbee::Match { + index: prev.index, + score: avg.min(u16::MAX as u32) as u16, + end_col: prev.end_col, // keep first part's position for filename bonus + exact: prev.exact && sm.exact, + } + }) + .collect(); + } + + matches +} + +/// Match + score across base and overflow files, each with their own arena. +#[tracing::instrument(skip_all, level = tracing::Level::DEBUG)] +pub(crate) fn fuzzy_match_and_score_files<'a>( + files: &'a [FileItem], + context: &ScoringContext, + base_count: usize, + base_arena: ArenaPtr, + overflow_arena: ArenaPtr, +) -> (Vec<&'a FileItem>, Vec, usize) { + // Process overflow files first: newly added files (created after the + // initial scan) live in the overflow arena and are more likely to be + // relevant to the current search. + // + // putting them first in the list makes sorting more efficient and gives + // them tiebreaker advantage in case sorting is the same + let results = if files.len() > base_count { + let mut results = match_and_score_in_arena(&files[base_count..], context, overflow_arena); + + results.extend(match_and_score_in_arena( + &files[..base_count], + context, + base_arena, + )); + + results + } else { + match_and_score_in_arena(files, context, base_arena) + }; + + sort_and_paginate(results, context) +} + +/// Resolve a DirItem's chunked path into frizbee's pointer buffer. +#[inline] +fn resolve_dir_chunks( + dir: &DirItem, + arena: ArenaPtr, + buf: &mut [*const u8; MAX_PATH_CHUNKS], +) -> Option<(usize, u16)> { + let ptrs = dir.path.resolve_ptrs(arena, buf); + Some((ptrs.len(), dir.path.byte_len)) +} + +/// Run SIMD fuzzy match across directory items, narrowing the candidate set +/// through each query part — same pipeline as file matching. +fn match_fuzzy_parts_dirs( + fuzzy_parts: &[&str], + working_dirs: &[&DirItem], + options: &neo_frizbee::Config, + max_threads: usize, + arena: ArenaPtr, +) -> Vec { + let valid_parts: Vec<&str> = fuzzy_parts + .iter() + .copied() + .filter(|p| p.len() >= 2) + .collect(); + + if valid_parts.is_empty() { + return vec![]; + } + + let resolve_chunks_for_frizbee = + |dir: &&DirItem, buf: &mut [*const u8; MAX_PATH_CHUNKS]| -> Option<(usize, u16)> { + resolve_dir_chunks(dir, arena, buf) + }; + + let first_part_matches = neo_frizbee::match_list_parallel_resolved( + valid_parts[0], + working_dirs, + &resolve_chunks_for_frizbee, + options, + max_threads, + ); + + if valid_parts.len() == 1 { + return first_part_matches; + } + + let mut matches = first_part_matches; + let total_parts = valid_parts.len() as u32; + for part in valid_parts[1..].iter() { + let mut part_options = *options; + part_options.max_typos = options.max_typos.map(|t| t.min(part.len() as u16)); + + // Collect the subset of dirs that survived the previous round. + let subset: Vec<&DirItem> = matches + .iter() + .map(|m| working_dirs[m.index as usize]) + .collect(); + + let sub_matches = neo_frizbee::match_list_parallel_resolved( + part, + subset.as_slice(), + &resolve_chunks_for_frizbee, + &part_options, + max_threads, + ); + + if sub_matches.is_empty() { + return vec![]; + } + + // Map sub_matches back to original indices, average scores across all parts. + matches = sub_matches + .into_iter() + .map(|sm| { + let prev = &matches[sm.index as usize]; + let sum = (prev.score as u32).saturating_add(sm.score as u32); + let avg = sum / total_parts; + + neo_frizbee::Match { + index: prev.index, + score: avg.min(u16::MAX as u32) as u16, + end_col: prev.end_col, // keep first part's position for dirname bonus + exact: prev.exact && sm.exact, + } + }) + .collect(); + } + + matches +} + +/// Match + score directories against a fuzzy query. +/// Scoring: base_score, frecency_boost, distance_penalty, dirname_bonus. +#[tracing::instrument(skip_all, level = tracing::Level::DEBUG)] +pub(crate) fn fuzzy_match_and_score_dirs<'a>( + dirs: &'a [DirItem], + context: &ScoringContext, + arena: ArenaPtr, +) -> (Vec<&'a DirItem>, Vec, usize) { + if dirs.is_empty() { + return (vec![], vec![], 0); + } + + let parsed_query = context.query; + let working_dirs: Vec<&DirItem> = if parsed_query.constraints.is_empty() { + dirs.iter().collect() + } else { + match apply_constraints(dirs, &parsed_query.constraints, arena, arena) { + Some(filtered) if !filtered.is_empty() => filtered, + Some(_) => return (vec![], vec![], 0), + None => dirs.iter().collect(), + } + }; + + let fuzzy_parts: &[&str] = match &parsed_query.fuzzy_query { + FuzzyQuery::Text(t) if t.len() >= 2 => std::slice::from_ref(t), + FuzzyQuery::Parts(parts) if !parts.is_empty() => parts.as_slice(), + _ => { + return score_dirs_by_frecency(&working_dirs, context); + } + }; + + let valid_parts: Vec<&str> = fuzzy_parts + .iter() + .copied() + .filter(|p| p.len() >= 2) + .collect(); + + if valid_parts.is_empty() { + return score_dirs_by_frecency(&working_dirs, context); + } + + let has_uppercase = valid_parts + .iter() + .any(|p| p.chars().any(|c| c.is_uppercase())); + + let options = neo_frizbee::Config { + max_typos: Some(context.max_typos), + sort: false, + scoring: Scoring { + capitalization_bonus: if has_uppercase { 8 } else { 0 }, + matching_case_bonus: if has_uppercase { 4 } else { 0 }, + ..Default::default() + }, + ..Default::default() + }; + + let path_matches = match_fuzzy_parts_dirs( + fuzzy_parts, + &working_dirs, + &options, + context.max_threads, + arena, + ); + + let main_needle = valid_parts[0].as_bytes(); + let main_needle_len = main_needle.len() as u16; + + let mut dir_buf = String::with_capacity(64); + let mut dirname_buf = String::with_capacity(32); + + let results: Vec<(&DirItem, Score)> = path_matches + .into_iter() + .map(|path_match| { + let dir = working_dirs[path_match.index as usize]; + let base_score = path_match.score as i32; + let frecency_boost = base_score.saturating_mul(dir.max_access_frecency()) / 100; + + // Distance penalty from current file's directory. + let distance_penalty = if context.current_file.is_some() { + dir.path.write_to_string(arena, &mut dir_buf); + calculate_distance_penalty(context.current_file, &dir_buf) + } else { + 0 + }; + + // Dirname bonus: if the match is in the last path segment. + let last_seg_offset = dir.last_segment_offset(); + let match_start_approx = path_match.end_col.saturating_sub(main_needle_len - 1); + let is_dirname_match = match_start_approx >= last_seg_offset; + + dir.write_dir_name(arena, &mut dirname_buf); + let dirname_len = dirname_buf.len(); + let is_exact_dirname = is_dirname_match + && main_needle_len as usize == dirname_len + && main_needle.eq_ignore_ascii_case(dirname_buf.as_bytes()); + + let filename_bonus = if is_exact_dirname { + base_score / 5 * 2 // 40% bonus for exact dirname match + } else if is_dirname_match { + base_score / 6 // ~16% bonus for fuzzy dirname match + } else { + 0 + }; + + let total = base_score + .saturating_add(frecency_boost) + .saturating_add(distance_penalty) + .saturating_add(filename_bonus); + + let score = Score { + total, + base_score, + filename_bonus, + special_filename_bonus: 0, + frecency_boost, + git_status_boost: 0, + distance_penalty, + current_file_penalty: 0, + combo_match_boost: 0, + path_alignment_bonus: 0, + exact_match: is_exact_dirname || path_match.exact, + match_type: if is_exact_dirname { + "exact_dirname" + } else if is_dirname_match { + "fuzzy_dirname" + } else if path_match.exact { + "exact_path" + } else { + "fuzzy_path" + }, + }; + + (dir, score) + }) + .collect(); + + sort_and_paginate_dirs(results, context) +} + +fn score_dirs_by_frecency<'a>( + dirs: &[&'a DirItem], + context: &ScoringContext, +) -> (Vec<&'a DirItem>, Vec, usize) { + let results: Vec<(&DirItem, Score)> = dirs + .iter() + .map(|&dir| { + let score = Score { + total: dir.max_access_frecency(), + frecency_boost: dir.max_access_frecency(), + match_type: "frecency", + ..Default::default() + }; + + (dir, score) + }) + .collect(); + + sort_and_paginate_dirs(results, context) +} + +/// Sort dir results by total score (descending) and apply pagination. +fn sort_and_paginate_dirs<'a>( + mut results: Vec<(&'a DirItem, Score)>, + context: &ScoringContext, +) -> (Vec<&'a DirItem>, Vec, usize) { + let total_matched = results.len(); + if total_matched == 0 { + return (vec![], vec![], 0); + } + + let offset = context.pagination.offset; + let limit = if context.pagination.limit > 0 { + context.pagination.limit + } else { + total_matched + }; + + if offset >= total_matched { + return (vec![], vec![], total_matched); + } + + let items_needed = offset.saturating_add(limit).min(total_matched); + let use_partial_sort = items_needed < total_matched / 2 && total_matched > 100; + + if use_partial_sort { + results.select_nth_unstable_by(items_needed - 1, |a, b| b.1.total.cmp(&a.1.total)); + results.truncate(items_needed); + } + + sort_with_buffer(&mut results, |a, b| b.1.total.cmp(&a.1.total)); + + if results.len() > limit { + let page_end = std::cmp::min(offset + limit, results.len()); + let page_size = page_end - offset; + results.drain(0..offset); + results.truncate(page_size); + } + + let (items, scores): (Vec<&DirItem>, Vec) = results.into_iter().unzip(); + (items, scores, total_matched) +} + +fn match_and_score_in_arena<'a>( + files: &'a [FileItem], + context: &ScoringContext, + arena: ArenaPtr, +) -> Vec<(&'a FileItem, Score)> { + if files.is_empty() { + return vec![]; + } + + let parsed = context.query; + let working_files: FileItems<'a> = if parsed.constraints.is_empty() { + FileItems::All(files) + } else { + match apply_constraints(files, &parsed.constraints, arena, arena) { + Some(filtered) if !filtered.is_empty() => FileItems::Filtered(filtered), + Some(_) => { + return vec![]; + } + None => FileItems::All(files), + } + }; + + let fuzzy_parts: &[&str] = match &parsed.fuzzy_query { + FuzzyQuery::Text(t) if t.len() >= 2 => std::slice::from_ref(t), + FuzzyQuery::Parts(parts) if !parts.is_empty() => parts.as_slice(), + _ => { + return score_filtered_by_frecency(&working_files, context, arena); + } + }; + + debug_assert!(!fuzzy_parts.is_empty()); + let has_uppercase = fuzzy_parts + .iter() + .any(|p| p.chars().any(|c| c.is_uppercase())); + // Users type `/` regardless of platform. Checking the OS separator alone + // would miss forward-slash queries on Windows. + let query_contains_path_separator = fuzzy_parts + .iter() + .any(|p| p.contains('/') || p.contains(MAIN_SEPARATOR)); + + let options = neo_frizbee::Config { + max_typos: Some(context.max_typos), + sort: false, + scoring: Scoring { + capitalization_bonus: if has_uppercase { 8 } else { 0 }, + matching_case_bonus: if has_uppercase { 4 } else { 0 }, + ..Default::default() + }, + ..Default::default() + }; + + let path_matches = match_fuzzy_parts( + fuzzy_parts, + &working_files, + &options, + context.max_threads, + arena, + ); + + let main_needle = fuzzy_parts[0].as_bytes(); // safe + let main_needle_len = main_needle.len() as u16; + + let mut fallback_indices: Vec = Vec::new(); + let filename_fallback_matches = if query_contains_path_separator || path_matches.len() > 15_000 + { + vec![] + } else { + let mut fallback_filenames: Vec> = Vec::new(); + + for (i, path_match) in path_matches.iter().enumerate() { + let file = working_files.index(path_match.index as usize); + let filename_start = file.filename_offset_in_relative_path() as u16; + let match_start_approx = path_match.end_col.saturating_sub(main_needle_len - 1); + + if match_start_approx < filename_start { + fallback_indices.push(i as u32); + fallback_filenames.push(file.path.filename_cow(arena)); + } + } + + if fallback_filenames.is_empty() { + vec![] + } else { + let mut matches = neo_frizbee::match_list_parallel( + fuzzy_parts[0], + &fallback_filenames, + &options, + if path_matches.len() > 4096 { + context.max_threads.div_ceil(2048) + } else { + 1 + }, + ); + + sort_by_key_with_buffer(&mut matches, |m| fallback_indices[m.index as usize]); + matches + } + }; + + let mut next_filename_match_cursor = 0; + let mut dir_buf = String::with_capacity(64); + let mut fname_buf = String::with_capacity(32); + let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE]; + + let results: Vec<_> = path_matches + .into_iter() + .enumerate() + .map(|(match_idx, path_match)| { + let file_idx = path_match.index as usize; + let file = working_files.index(file_idx); + + let base_score = path_match.score as i32; + let frecency_boost = base_score.saturating_mul(file.total_frecency_score()) / 100; + + let git_status_boost = if file.git_status.is_some_and(is_modified_status) { + base_score * 15 / 100 + } else { + 0 + }; + + if context.current_file.is_some() || context.last_same_query_match.is_some() { + file.write_dir_str(arena, &mut dir_buf); + } + + let distance_penalty = if context.current_file.is_some() { + calculate_distance_penalty(context.current_file, &dir_buf) + } else { + 0 + }; + + let filename_start = file.filename_offset_in_relative_path() as u16; + let match_start_approx = path_match.end_col.saturating_sub(main_needle_len - 1); + + let end_col_filename_match = match_start_approx >= filename_start; + let simd_filename_match = if !end_col_filename_match { + filename_fallback_matches + .get(next_filename_match_cursor) + .and_then(|m| { + if fallback_indices[m.index as usize] == match_idx as u32 { + next_filename_match_cursor += 1; + Some(m) + } else { + None + } + }) + } else { + None + }; + + let is_filename_match = end_col_filename_match || simd_filename_match.is_some(); + let fname_len = file.path.byte_len as usize - file.path.filename_offset as usize; + + let is_exact_filename = simd_filename_match.is_some_and(|m| m.exact) + || (end_col_filename_match && main_needle_len as usize == fname_len && { + file.write_file_name_from_arena(arena, &mut fname_buf); + main_needle.eq_ignore_ascii_case(fname_buf.as_bytes()) + }); + + let mut has_special_filename_bonus = false; + let filename_bonus = if is_exact_filename { + base_score / 5 * 2 // 40% bonus for exact filename match + } else if is_filename_match { + // 16% bonus for fuzzy filename match that landed in the filename region. + // For fallback matches (where the path match landed in a directory segment), + // scale the bonus by the quality of the filename match — a contiguous match + // like "rename" in "rename.ts" gets the full bonus, while a scattered + // subsequence like r-e-n-a-m-e in "generateSessionName.ts" gets much less. + let max_bonus = (base_score / 6).min(30); + if let Some(fm) = simd_filename_match { + let max_possible = main_needle_len as i32 * 16; + let quality = (fm.score as i32).min(max_possible); + max_bonus * quality / max_possible + } else { + max_bonus + } + } else if !is_filename_match && (5..=11).contains(&fname_len) { + file.write_file_name_from_arena(arena, &mut fname_buf); + // 5% bonus for special file but not as much as file name to avoid situations + // when you have /user_service/server.rs and /user_service/server/mod.rs + if is_special_entry_point_file(&fname_buf) { + has_special_filename_bonus = true; + base_score * 5 / 100 + } else { + 0 + } + } else { + 0 + }; + + let current_file_penalty = + calculate_current_file_penalty(file, base_score / 4, context, arena); + let combo_match_boost = { + let last_same_query_match = context.last_same_query_match.as_ref().filter(|m| { + let file_path_str = m.file_path.to_string_lossy(); + let total_len = file.path.byte_len as usize; + if file_path_str.len() < total_len { + return false; + } + // Reuse dir_buf (already has capacity) for the full path + file.write_relative_path_from_arena(arena, &mut dir_buf); + file_path_str.ends_with(dir_buf.as_str()) + }); + + match last_same_query_match { + Some(_) if context.min_combo_count == 0 => 1000, + Some(combo_match) if combo_match.open_count >= context.min_combo_count => { + combo_match.open_count as i32 * context.combo_boost_score_multiplier + } + Some(combo_match) => combo_match.open_count as i32 * 5, + _ => 0, + } + }; + + // Path alignment bonus: when the query looks like a file path, + // reward candidates whose path closely matches the typed query. + // Uses suffix overlap — bytes matching from the end. A full prefix + // match is just the 100% coverage case, so no separate branch needed. + let path_alignment_bonus = if query_contains_path_separator { + let rel_path = file.path.read_to_buf(arena, &mut path_buf); + let path_bytes = rel_path.as_bytes(); + let common_suffix = main_needle + .iter() + .rev() + .zip(path_bytes.iter().rev()) + .take_while(|(n, p): &(&u8, &u8)| n.eq_ignore_ascii_case(p)) + .count(); + + let needle_len = main_needle.len(); + if common_suffix > 10 && needle_len > 0 { + let coverage = common_suffix * 100 / needle_len; + if coverage >= 30 { + base_score * coverage as i32 / 100 + } else { + 0 + } + } else { + 0 + } + } else { + 0 + }; + + let total = base_score + .saturating_add(frecency_boost) + .saturating_add(git_status_boost) + .saturating_add(distance_penalty) + .saturating_add(filename_bonus) + .saturating_add(current_file_penalty) + .saturating_add(combo_match_boost) + .saturating_add(path_alignment_bonus); + + let score = Score { + total, + base_score, + current_file_penalty, + filename_bonus, + special_filename_bonus: if has_special_filename_bonus { + filename_bonus + } else { + 0 + }, + frecency_boost, + git_status_boost, + distance_penalty, + combo_match_boost, + path_alignment_bonus, + exact_match: is_exact_filename || path_match.exact, + match_type: if is_exact_filename { + "exact_filename" + } else if is_filename_match { + "fuzzy_filename" + } else if path_match.exact { + "exact_path" + } else { + "fuzzy_path" + }, + }; + + (file, score) + }) + .collect(); + + results +} + +fn is_special_entry_point_file(filename: &str) -> bool { + matches!( + filename, + "mod.rs" + | "lib.rs" + | "main.rs" + | "index.js" + | "index.jsx" + | "index.ts" + | "index.tsx" + | "index.mjs" + | "index.cjs" + | "index.vue" + | "__init__.py" + | "__main__.py" + | "main.go" + | "main.c" + | "index.php" + | "main.rb" + | "index.rb" + ) +} + +fn score_filtered_by_frecency<'a>( + files: &FileItems<'a>, + context: &ScoringContext, + arena: ArenaPtr, +) -> Vec<(&'a FileItem, Score)> { + let score_file = |file: &'a FileItem| { + let total_frecency_score = file.access_frecency_score as i32 + + (file.modification_frecency_score as i32).saturating_mul(4); + + // Give modified/dirty files a boost even in frecency-only mode + let git_status_boost = if file.git_status.is_some_and(is_modified_status) { + total_frecency_score * 15 / 100 + } else { + 0 + }; + + let current_file_penalty = + calculate_current_file_penalty(file, total_frecency_score, context, arena); + let total = total_frecency_score + .saturating_add(git_status_boost) + .saturating_add(current_file_penalty); + + let score = Score { + total, + base_score: 0, + filename_bonus: 0, + distance_penalty: 0, + special_filename_bonus: 0, + combo_match_boost: 0, + path_alignment_bonus: 0, + current_file_penalty, + frecency_boost: total_frecency_score, + git_status_boost, + exact_match: false, + match_type: "frecency", + }; + + (file, score) + }; + + match files { + FileItems::All(s) => s + .par_iter() + .filter_map(|f| { + let live = !f.is_deleted(); + live.then_some(score_file(f)) + }) + .collect(), + FileItems::Filtered(v) => v + .iter() + .filter_map(|f| { + let live = !f.is_deleted(); + live.then_some(score_file(f)) + }) + .collect(), + } +} + +#[inline] +fn calculate_current_file_penalty( + file: &FileItem, + base_score: i32, + context: &ScoringContext, + arena: ArenaPtr, +) -> i32 { + let mut penalty = 0i32; + + if let Some(current) = context.current_file + && file.relative_path_eq(arena, current) + { + penalty -= base_score; + } + + penalty +} + +/// Sorts elements by total score (descending) and returns the requested page. +/// Always returns results in descending order (best scores first). +/// The UI layer handles rendering order based on prompt position. +#[tracing::instrument(skip_all, level = tracing::Level::DEBUG)] +fn sort_and_paginate<'a>( + mut results: Vec<(&'a FileItem, Score)>, + context: &ScoringContext, +) -> (Vec<&'a FileItem>, Vec, usize) { + let total_matched = results.len(); + + if total_matched == 0 { + return (vec![], vec![], 0); + } + + let offset = context.pagination.offset; + let limit = if context.pagination.limit > 0 { + context.pagination.limit + } else { + total_matched + }; + + // Check if offset is out of bounds + if offset >= total_matched { + tracing::warn!( + offset = offset, + total_matched = total_matched, + "Pagination: offset >= total_matched, returning empty" + ); + + return (vec![], vec![], total_matched); + } + + let items_needed = offset.saturating_add(limit).min(total_matched); + // Use partial sort if we need less than half the results and dataset is large + let use_partial_sort = items_needed < total_matched / 2 && total_matched > 100; + // Always sort in descending order (best scores first) + if use_partial_sort { + // Partition at position (items_needed - 1) with descending comparator + // This puts the highest N needed items at the front + results.select_nth_unstable_by(items_needed - 1, |a, b| { + b.1.total + .cmp(&a.1.total) + .then_with(|| b.0.modified.cmp(&a.0.modified)) + }); + results.truncate(items_needed); + } + + // select nth does not sort the results, we have to sort accordingly anyway + sort_with_buffer(&mut results, |a, b| { + b.1.total + .cmp(&a.1.total) + .then_with(|| b.0.modified.cmp(&a.0.modified)) + }); + + // in the best scenario truncation happened in the select_nth step + if results.len() > limit { + let page_end = std::cmp::min(offset + limit, results.len()); + let page_size = page_end - offset; + + results.drain(0..offset); + results.truncate(page_size); + } + + let (items, scores): (Vec<&FileItem>, Vec) = results.into_iter().unzip(); + (items, scores, total_matched) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::PaginationArgs; + use fff_query_parser::QueryParser; + + fn make_test_files(specs: &[(&str, i32, u64)]) -> (Vec<(FileItem, Score)>, ArenaPtr) { + let path_strings: Vec = specs.iter().map(|(p, _, _)| p.to_string()).collect(); + let items: Vec = specs + .iter() + .map(|(p, _, _)| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let result: Vec<(FileItem, Score)> = specs + .iter() + .enumerate() + .map(|(i, &(_, score, modified))| { + let filename_start = path_strings[i] + .rfind(std::path::is_separator) + .map(|j| j + 1) + .unwrap_or(0) as u16; + let mut file = FileItem::new_raw(filename_start, 0, modified, None, false); + file.set_path(strings[i].clone()); + let score_obj = Score { + total: score, + base_score: score, + filename_bonus: 0, + distance_penalty: 0, + special_filename_bonus: 0, + current_file_penalty: 0, + frecency_boost: 0, + git_status_boost: 0, + exact_match: false, + match_type: "test", + combo_match_boost: 0, + path_alignment_bonus: 0, + }; + (file, score_obj) + }) + .collect(); + std::mem::forget(store); + (result, arena) + } + + #[test] + fn test_partial_sort_descending() { + // Create test data with known scores + let (test_data, arena) = make_test_files(&[ + ("file1.rs", 100, 1000), + ("file2.rs", 200, 2000), + ("file3.rs", 50, 3000), + ("file4.rs", 300, 4000), + ("file5.rs", 150, 5000), + ("file6.rs", 250, 6000), + ("file7.rs", 80, 7000), + ("file8.rs", 180, 8000), + ("file9.rs", 120, 9000), + ("file10.rs", 90, 10000), + ]); + + // Convert to references like the actual function uses + let results: Vec<(&FileItem, Score)> = test_data + .iter() + .map(|(file, score)| (file, score.clone())) + .collect(); + + let query_str = "test"; + let parser = QueryParser::default(); + let query = parser.parse(query_str); + let context = ScoringContext { + query: &query, + max_threads: 1, + max_typos: 2, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + + pagination: PaginationArgs { + offset: 0, + limit: 0, + }, + }; + + // Test with full sort - returns all results sorted descending + let (items, scores, total) = sort_and_paginate(results.clone(), &context); + + // Should return all 10 items sorted by score descending + assert_eq!(total, 10); + assert_eq!(scores.len(), 10); + assert_eq!(scores[0].total, 300, "First should be highest score"); + assert_eq!(scores[1].total, 250, "Second should be second highest"); + assert_eq!(scores[2].total, 200, "Third should be third highest"); + + // Verify the files match + assert_eq!(items[0].relative_path(arena), "file4.rs"); + assert_eq!(items[1].relative_path(arena), "file6.rs"); + assert_eq!(items[2].relative_path(arena), "file2.rs"); + } + + #[test] + fn test_partial_sort_with_same_scores() { + // Test tiebreaker with modified time + let (test_data, _arena) = make_test_files(&[ + ("file1.rs", 100, 5000), // Same score, older + ("file2.rs", 100, 8000), // Same score, newer + ("file3.rs", 100, 3000), // Same score, oldest + ("file4.rs", 200, 1000), + ("file5.rs", 200, 9000), // Higher score, newest + ]); + + let results: Vec<(&FileItem, Score)> = test_data + .iter() + .map(|(file, score)| (file, score.clone())) + .collect(); + + let query_str = "test"; + let parser = QueryParser::default(); + let query = parser.parse(query_str); + let context = ScoringContext { + query: &query, + max_threads: 1, + max_typos: 2, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + + pagination: PaginationArgs { + offset: 0, + limit: 0, + }, + }; + + let (items, scores, _) = sort_and_paginate(results, &context); + + // Should return all 5 items sorted: 200(9000), 200(1000), 100(8000), 100(5000), 100(3000) + assert_eq!(scores.len(), 5); + assert_eq!(scores[0].total, 200); + assert_eq!(items[0].modified, 9000, "First 200 should be newest"); + assert_eq!(scores[1].total, 200); + assert_eq!(items[1].modified, 1000, "Second 200 should be older"); + assert_eq!(scores[2].total, 100); + assert_eq!(items[2].modified, 8000, "First 100 should be newest"); + assert_eq!(scores[3].total, 100); + assert_eq!(items[3].modified, 5000); + assert_eq!(scores[4].total, 100); + assert_eq!(items[4].modified, 3000, "Last 100 should be oldest"); + } + + #[test] + fn test_no_partial_sort_for_small_results() { + // When results.len() <= threshold, should use regular sort + let (test_data, arena) = make_test_files(&[ + ("file1.rs", 100, 1000), + ("file2.rs", 200, 2000), + ("file3.rs", 50, 3000), + ]); + + let results: Vec<(&FileItem, Score)> = test_data + .iter() + .map(|(file, score)| (file, score.clone())) + .collect(); + + let query_str = "test"; + let parser = QueryParser::default(); + let query = parser.parse(query_str); + let context = ScoringContext { + query: &query, + max_threads: 1, + max_typos: 2, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + + pagination: PaginationArgs { + offset: 0, + limit: 0, + }, + }; + + // Returns all results sorted descending + let (items, scores, _) = sort_and_paginate(results, &context); + + assert_eq!(scores.len(), 3); + assert_eq!(scores[0].total, 200); + assert_eq!(scores[1].total, 100); + assert_eq!(scores[2].total, 50); + assert_eq!(items[0].relative_path(arena), "file2.rs"); + assert_eq!(items[1].relative_path(arena), "file1.rs"); + assert_eq!(items[2].relative_path(arena), "file3.rs"); + } +} + +#[cfg(test)] +mod filename_bonus_tests { + use super::*; + use crate::types::PaginationArgs; + use fff_query_parser::QueryParser; + fn make_files(paths: &[&str]) -> (Vec, ArenaPtr) { + let path_strings: Vec = paths.iter().map(|p| p.to_string()).collect(); + let items: Vec = paths + .iter() + .map(|p| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let mut result: Vec = items; + for (i, file) in result.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + } + std::mem::forget(store); + (result, arena) + } + + fn make_files_with_frecency(specs: &[(&str, i16)]) -> (Vec, ArenaPtr) { + let path_strings: Vec = specs.iter().map(|(p, _)| p.to_string()).collect(); + let items: Vec = specs + .iter() + .map(|(p, _)| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let mut result: Vec = items; + for (i, file) in result.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + file.access_frecency_score = specs[i].1; + } + std::mem::forget(store); + (result, arena) + } + + /// Run `fuzzy_match_and_score_files` with production-like max_typos scaling. + fn search(files: &[FileItem], query: &str, arena: ArenaPtr) -> Vec<(String, Score)> { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + + let effective_query = match &parsed.fuzzy_query { + FuzzyQuery::Text(t) => *t, + FuzzyQuery::Parts(parts) if !parts.is_empty() => parts[0], + _ => query, + }; + let max_typos = (effective_query.len() as u16 / 4).clamp(2, 6); + + let ctx = ScoringContext { + query: &parsed, + max_threads: 1, + max_typos, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }; + let (items, scores, _) = + fuzzy_match_and_score_files(files, &ctx, files.len(), arena, arena); + items + .iter() + .zip(scores.iter()) + .map(|(f, s)| (f.relative_path(arena).to_string(), s.clone())) + .collect() + } + + #[test] + fn test_filename_match_ranks_above_path_only_match() { + let (files, arena) = make_files(&["src/username/handler.rs", "src/username/username.rs"]); + + let results = search(&files, "usrnmea", arena); + + assert!( + results.len() >= 2, + "both files should match, got {}", + results.len() + ); + assert_eq!( + results[0].0, "src/username/username.rs", + "filename match should rank first" + ); + assert!( + results[0].1.filename_bonus > 0, + "username.rs should have filename bonus" + ); + assert_eq!( + results[1].1.filename_bonus, 0, + "handler.rs should have no filename bonus" + ); + } + + #[test] + fn test_exact_filename_beats_fuzzy_filename() { + let (files, arena) = make_files(&["src/user_name_handler.rs", "src/username.rs"]); + + let results = search(&files, "username.rs", arena); + + assert!(results.len() >= 2); + assert_eq!( + results[0].0, "src/username.rs", + "exact filename should rank first" + ); + assert_eq!(results[0].1.match_type, "exact_filename"); + assert!(results[0].1.filename_bonus > results[1].1.filename_bonus); + } + + #[test] + fn test_same_length_filename_no_false_exact() { + let (files, arena) = make_files(&["src/item_sync/file.rs", "src/models/item.rs"]); + + let results = search(&files, "item.rs", arena); + + assert!(results.len() >= 2); + assert_eq!(results[0].0, "src/models/item.rs"); + assert_eq!(results[0].1.match_type, "exact_filename"); + assert_ne!( + results[1].1.match_type, "exact_filename", + "file.rs should not get exact_filename" + ); + } + + #[test] + fn test_path_separator_disables_filename_bonus() { + let (files, arena) = make_files(&["src/controllers/user.rs"]); + + let results = search(&files, "src/user", arena); + + assert!(!results.is_empty()); + assert_eq!( + results[0].1.filename_bonus, 0, + "path-like query should not get filename bonus" + ); + } + + /// Regression: full-path query should rank the near-exact path match first. + /// https://x.com/mbarneyjr/status/2043474268390817861 + #[test] + fn test_full_path_query_prefers_closer_filename_match() { + let (files, arena) = make_files_with_frecency(&[ + ( + "test-utils/completion/condition-key/yaml_partial-svc-colon.yml", + 0, + ), + ( + "test-utils/test-cases/completion/condition-key/yaml_partial-svc.yml", + 0, + ), + ( + "test-utils/action-value/yaml_inline_partial-svc-colon.yml", + 0, + ), + ( + "test-utils/completion/action-value/yaml_array_partial-svc-colon.yml", + 0, + ), + ( + "test-utils/completion/action-value/yaml_array_partial-svc.yml", + 0, + ), + ( + "test-utils/completion/condition-key/yaml_global-tag-keys.yml", + 0, + ), + ( + "test-utils/test-cases/completion/condition-key/yaml_partial.yml", + 10, + ), + ]); + + let results = search( + &files, + "t-utils/test-cases/completion/condition-key/yaml_partial-svc.yml", + arena, + ); + + assert!(!results.is_empty(), "query should match at least one file"); + + assert_eq!( + results[0].0, "test-utils/test-cases/completion/condition-key/yaml_partial-svc.yml", + "near-exact full-path match should rank first, but got: {} \ + (total={}, base={}, frecency={})", + results[0].0, results[0].1.total, results[0].1.base_score, results[0].1.frecency_boost, + ); + } + + /// Regression: PR #652 / field panic in pi-fff v0.9.6. + /// A path >512 bytes (but within PATH_MAX) overflows the fixed + /// `[*const u8; 32]` chunk-pointer buffer during scoring and panics with + /// "index out of bounds: the len is 32 but the index is 32". + #[test] + fn test_path_longer_than_512_bytes_does_not_panic_and_matches() { + let mut long_path = String::new(); + while long_path.len() < 600 { + long_path.push_str("deeply_nested_directory_segment/"); + } + long_path.push_str("needle_file.rs"); + assert!(long_path.len() > 512 && long_path.len() < crate::simd_path::PATH_BUF_SIZE); + + let (files, arena) = make_files(&[long_path.as_str(), "src/other.rs"]); + + // Panics here on unfixed code: frizbee resolves chunk ptrs per file. + let results = search(&files, "needle", arena); + + assert!( + results.iter().any(|(p, _)| p == &long_path), + "filename at the tail of a >512-byte path must still match, got: {:?}", + results.iter().map(|(p, _)| p).collect::>() + ); + } + + #[test] + fn test_single_path_matching() { + let path = "core_workflow_service/kafka_event_consumer/src/ai_part_extraction_request/ai_part_extraction_request_handler.rs"; + + let options = neo_frizbee::Config { + max_typos: Some(2), + sort: false, + ..Default::default() + }; + + let matches = neo_frizbee::match_list("aipart", &[path], &options); + assert!(!matches.is_empty(), "'aipart' should match the path"); + + let matches = neo_frizbee::match_list("core", &[path], &options); + assert!(!matches.is_empty(), "'core' should match the path"); + + let co_options = neo_frizbee::Config { + max_typos: Some(2), + ..options + }; + let matches = neo_frizbee::match_list("co", &[path], &co_options); + assert!(!matches.is_empty(), "'co' should match the path"); + } + + #[test] + fn test_lowercase_path_matching() { + let path = "core_workflow_service/kafka_event_consumer/src/ai_part_extraction_request/ai_part_extraction_request_handler.rs".to_lowercase(); + + let options = neo_frizbee::Config { + max_typos: Some(2), + sort: false, + ..Default::default() + }; + + let matches = neo_frizbee::match_list("co", &[path.as_str()], &options); + assert!(!matches.is_empty(), "'co' should match the lowercase path"); + + let matches = neo_frizbee::match_list("core", &[path.as_str()], &options); + assert!( + !matches.is_empty(), + "'core' should match the lowercase path" + ); + } +} + +#[cfg(test)] +mod typo_resistance_tests { + use super::*; + use crate::types::PaginationArgs; + use fff_query_parser::QueryParser; + + fn make_files(paths: &[&str]) -> (Vec, ArenaPtr) { + let path_strings: Vec = paths.iter().map(|p| p.to_string()).collect(); + let items: Vec = paths + .iter() + .map(|p| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let mut result: Vec = items; + for (i, file) in result.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + } + std::mem::forget(store); + (result, arena) + } + + fn search_with_typos( + files: &[FileItem], + query: &str, + arena: ArenaPtr, + max_typos: u16, + ) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let ctx = ScoringContext { + query: &parsed, + max_threads: 1, + max_typos, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }; + let (items, _, _) = fuzzy_match_and_score_files(files, &ctx, files.len(), arena, arena); + items.iter().map(|f| f.relative_path(arena)).collect() + } + + #[test] + fn test_typo_resistant_long_query() { + let (files, arena) = make_files(&[ + "src/pricing/bid_comparison_supplier_part_cost_modifiers.rs", + "src/pricing/bid_evaluation_handler.rs", + "src/pricing/supplier_contract_terms.rs", + "src/models/purchase_order.rs", + "src/models/inventory_item.rs", + "src/controllers/user_controller.rs", + "src/controllers/admin_controller.rs", + "src/services/notification_service.rs", + "src/services/email_dispatcher.rs", + "src/utils/string_helpers.rs", + "src/utils/date_formatter.rs", + "src/db/migration_runner.rs", + "src/db/connection_pool.rs", + "src/config/app_settings.rs", + "src/config/feature_flags.rs", + ]); + + // Exact substring — must always match + let results = search_with_typos(&files, "bid_comparison", arena, 6); + assert!( + results + .iter() + .any(|p| p.contains("bid_comparison_supplier_part_cost_modifiers")), + "exact substring 'bid_comparison' should match, got: {results:?}" + ); + + // Concatenated query with typos — the real-world scenario. + // "bidcomparsionsupplierpartcostmodfiers" is a smushed version of + // "bid_comparison_supplier_part_cost_modifiers" with 2 typos + // (comparison → comparison, modifiers → modifiers). + let results = search_with_typos(&files, "bidcomparsionsupplierpartcostmodfiers", arena, 6); + assert!( + results + .iter() + .any(|p| p.contains("bid_comparison_supplier_part_cost_modifiers")), + "typo query 'bidcomparsionsupplierpartcostmodfiers' should match, got: {results:?}" + ); + + // Shorter typo query + let results = search_with_typos(&files, "bidcomp", arena, 4); + assert!( + results.iter().any(|p| p.contains("bid_comparison")), + "'bidcomp' should match bid_comparison file, got: {results:?}" + ); + + // Query with missing underscores + let results = search_with_typos(&files, "supplierpartcost", arena, 6); + assert!( + results + .iter() + .any(|p| p.contains("bid_comparison_supplier_part_cost_modifiers")), + "'supplierpartcost' should match, got: {results:?}" + ); + } +} + +#[cfg(test)] +mod constraint_only_query_tests { + use super::*; + use crate::types::PaginationArgs; + use fff_query_parser::QueryParser; + + #[test] + fn constraint_only_git_status_query_returns_filtered_files() { + let paths = ["modified.rs", "clean.rs"]; + let path_strings: Vec = paths.iter().map(|p| p.to_string()).collect(); + let items: Vec = paths + .iter() + .map(|p| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let mut files: Vec = items; + for (i, file) in files.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + } + files[0].git_status = Some(git2::Status::WT_MODIFIED); + files[1].git_status = Some(git2::Status::CURRENT); + std::mem::forget(store); + + let parser = QueryParser::default(); + let parsed = parser.parse("git:modified"); + + let ctx = ScoringContext { + query: &parsed, + max_threads: 1, + max_typos: 2, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 50, + }, + }; + + let (items, _scores, total_matched) = + fuzzy_match_and_score_files(&files, &ctx, files.len(), arena, arena); + + assert_eq!( + total_matched, 1, + "git:modified constraint-only query should match exactly 1 modified file" + ); + assert_eq!(items.len(), 1); + assert_eq!(items[0].relative_path(arena), "modified.rs"); + } + + #[test] + fn constraint_only_status_modified_alias_returns_filtered_files() { + let paths = ["a.rs", "b.rs"]; + let path_strings: Vec = paths.iter().map(|p| p.to_string()).collect(); + let items: Vec = paths + .iter() + .map(|p| { + let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16; + FileItem::new_raw(fname, 0, 0, None, false) + }) + .collect(); + let (store, strings) = + crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items); + let arena = store.as_arena_ptr(); + let mut files: Vec = items; + for (i, file) in files.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + } + files[0].git_status = Some(git2::Status::WT_MODIFIED); + files[1].git_status = Some(git2::Status::CURRENT); + std::mem::forget(store); + + let parser = QueryParser::default(); + let parsed = parser.parse("status:modified"); + + let ctx = ScoringContext { + query: &parsed, + max_threads: 1, + max_typos: 2, + current_file: None, + last_same_query_match: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 50, + }, + }; + + let (items, _scores, total_matched) = + fuzzy_match_and_score_files(&files, &ctx, files.len(), arena, arena); + + assert_eq!(total_matched, 1); + assert_eq!(items[0].relative_path(arena), "a.rs"); + } +} diff --git a/crates/fff-core/src/shared.rs b/crates/fff-core/src/shared.rs new file mode 100644 index 0000000..10d6e65 --- /dev/null +++ b/crates/fff-core/src/shared.rs @@ -0,0 +1,391 @@ +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak}; +use std::time::{Duration, Instant}; + +use crate::dbs::lmdb::{LmdbStore, spawn_lmdb_gc}; +use crate::error::Error; +use crate::file_picker::FilePicker; +use crate::frecency::FrecencyTracker; +use crate::git::GitStatusCache; +use crate::query_tracker::QueryTracker; +use crate::scan::ScanJob; +use git2::Repository; + +/// Poll `.git/index.lock` until it disappears (git write completed), giving up +/// after [`GIT_LOCK_MAX_WAIT`]. Used by [`SharedPicker::refresh_git_status`] +/// to avoid reading a half-updated index when the watcher fires mid-`git add`. +/// +/// The wait is bounded and cheap: the lock file is typically cleared within +/// a few milliseconds of the git command exiting. +fn wait_for_git_index_lock_release(git_root: &Path) { + const GIT_LOCK_POLL: Duration = Duration::from_millis(10); + const GIT_LOCK_MAX_WAIT: Duration = Duration::from_millis(500); + + let lock = git_root.join(".git").join("index.lock"); + // Fast path: no lock present. + if !lock.exists() { + return; + } + let deadline = Instant::now() + GIT_LOCK_MAX_WAIT; + while lock.exists() && Instant::now() < deadline { + std::thread::sleep(GIT_LOCK_POLL); + } + if lock.exists() { + tracing::warn!( + "Proceeding with git status refresh despite lingering \ + .git/index.lock at {} — will retry once it clears", + lock.display() + ); + } +} + +/// Poll `done` every 10ms until it returns `true`, or until `timeout` elapses. +/// Returns `true` if the condition was met, `false` on timeout. +fn poll_until(timeout: Duration, mut done: impl FnMut() -> bool) -> bool { + let start = Instant::now(); + while !done() { + if start.elapsed() >= timeout { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } + true +} + +/// Thread-safe shared handle to the [`FilePicker`] instance. +/// This accumulates only asynchronous non-blocking operations against the +/// file picker: creating, triggering various rescans and so on. +/// +/// For blocking access use internal picker via `.read()` or `.write()` +/// +/// ```ignore +/// let shared_picker = SharedFilePicker::default(); +/// +/// if let Some(picker) = shared_picker.read()?.as_ref() { +/// let files = picker.fuzzy_search(&query, options); +/// println!("Found {} files", files.len()); +/// } else { +/// println!("Picker not initialized"); +/// } +/// ``` +#[derive(Clone, Default)] +pub struct SharedFilePicker(pub(crate) Arc); + +pub struct SharedPickerInner { + picker: parking_lot::RwLock>, +} + +impl Default for SharedPickerInner { + fn default() -> Self { + Self { + picker: parking_lot::RwLock::new(None), + } + } +} + +/// Non-owning handle to a [`SharedPicker`]. +#[derive(Clone)] +pub(crate) struct WeakFilePicker(Weak); + +impl WeakFilePicker { + /// Try to promote the weak handle back to a strong [`SharedPicker`]. + /// + /// Returns `None` once every strong `SharedPicker` clone has been + /// dropped. Callers should treat that as "the picker is being + /// torn down" and exit their current iteration cleanly. + pub(crate) fn upgrade(&self) -> Option { + self.0.upgrade().map(SharedFilePicker) + } +} + +impl std::fmt::Debug for SharedFilePicker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SharedPicker").field(&"..").finish() + } +} + +impl SharedFilePicker { + pub fn read(&self) -> Result>, Error> { + Ok(self.0.picker.read()) + } + + pub fn write(&self) -> Result>, Error> { + Ok(self.0.picker.write()) + } + + /// Signal the background scan to cancel. Non-blocking: post-scan + /// threads check this flag and bail out at their next cancellation point. + pub fn cancel(&self) { + if let Ok(guard) = self.read() + && let Some(picker) = guard.as_ref() + { + picker.cancel(); + } + } + + /// Produce a non-owning handle to the same inner picker. + /// Use it if you don't need to block internal threads from dropping while owning this ref + pub(crate) fn weaken(&self) -> WeakFilePicker { + WeakFilePicker(Arc::downgrade(&self.0)) + } + + /// Return `true` if this is an instance of the picker that requires a complicated post-scan + /// indexing/cache warmup job. The indexing is not crazy but it takes time. + pub fn need_complex_rebuild(&self) -> bool { + let guard = self.0.picker.read(); + guard + .as_ref() + .is_some_and(|p| p.has_mmap_cache() || p.has_content_indexing()) + } + + /// Block until the background filesystem scan finishes. + /// Returns `true` if scan completed, `false` on timeout. + pub fn wait_for_scan(&self, timeout: Duration) -> bool { + let signal = { + let guard = self.0.picker.read(); + match &*guard { + Some(picker) => Arc::clone(&picker.signals.scanning), + None => return true, + } + }; + + poll_until(timeout, || { + !signal.load(std::sync::atomic::Ordering::Acquire) + }) + } + + /// Block until the background file watcher is ready. + /// Returns `true` if watcher ready, `false` on timeout. + pub fn wait_for_watcher(&self, timeout: Duration) -> bool { + let watch_ready_signal = { + let guard = self.0.picker.read(); + match &*guard { + Some(picker) => Arc::clone(&picker.signals.watcher_ready), + None => return true, + } + }; + + poll_until(timeout, || { + watch_ready_signal.load(std::sync::atomic::Ordering::Acquire) + }) + } + + /// Blocks until both the filesystem walk and post-scan indexing are done. + /// Returns true once scanning=false AND post_scan_indexing_active=false. + pub fn wait_for_indexing_complete(&self, timeout: Duration) -> bool { + let (scanning, post_scan_active) = { + let guard = self.0.picker.read(); + match &*guard { + Some(picker) => ( + Arc::clone(&picker.signals.scanning), + Arc::clone(&picker.signals.post_scan_indexing_active), + ), + None => return true, + } + }; + + poll_until(timeout, || { + !scanning.load(std::sync::atomic::Ordering::Acquire) + && !post_scan_active.load(std::sync::atomic::Ordering::Acquire) + }) + } + + /// Trigger a full filesystem rescan without blocking the caller. + /// Performs a safe async rescan. Guarantees only single active rescan per picker. + /// If many rescans requested the last one guaranteed to be finished. + pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> { + match ScanJob::new_rescan(self, shared_frecency)? { + Some(job) => { + job.spawn(); + } + None => { + // we can not abort the ongoing sync, but if the events + if let Ok(guard) = self.read() + && let Some(picker) = guard.as_ref() + { + picker + .scan_signals() + .rescan_pending + .store(true, std::sync::atomic::Ordering::Release); + tracing::info!( + "Full rescan requested while another scan is active — \ + deferred via rescan_pending flag" + ); + } + } + } + Ok(()) + } + + /// Refresh git statuses for all indexed files + #[tracing::instrument(level = "info", skip_all)] + pub fn refresh_git_status(&self, shared_frecency: &SharedFrecency) -> Result { + use tracing::debug; + + let git_status = { + let git_root = { + let guard = self.read()?; + let Some(ref picker) = *guard else { + return Err(Error::FilePickerMissing); + }; + picker.git_root().map(|p| p.to_path_buf()) + }; + + debug!(?git_root, "Refreshing git status for picker"); + + if let Some(ref root) = git_root { + wait_for_git_index_lock_release(root); + } + + GitStatusCache::read_git_status( + git_root.as_deref(), + &mut crate::git::default_status_options(), + ) + }; + + let mut guard = self.write()?; + let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?; + + let statuses_count = if let Some(git_status) = git_status { + let count = git_status.statuses_len(); + picker.update_git_statuses(git_status, shared_frecency)?; + count + } else { + 0 + }; + + Ok(statuses_count) + } + + /// Recompute and apply git status for a specific set of paths. + pub fn update_git_status_for_paths( + &self, + paths: &[PathBuf], + shared_frecency: &SharedFrecency, + ) -> Result<(), Error> { + if paths.is_empty() { + return Ok(()); + } + + let git_root = { + let guard = self.read()?; + let Some(ref picker) = *guard else { + return Err(Error::FilePickerMissing); + }; + picker.git_root().map(|p| p.to_path_buf()) + }; + let Some(git_root) = git_root else { + return Ok(()); + }; + + wait_for_git_index_lock_release(&git_root); + + let repo = Repository::open(&git_root)?; + let status = GitStatusCache::git_status_for_paths(&repo, paths)?; + + let mut guard = self.write()?; + let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?; + picker.update_git_statuses(status, shared_frecency) + } +} + +/// Thread-safe shared handle to an LMDB-backed store. A disabled (`noop`) +/// instance silently ignores writes. See the [`SharedFrecency`] and +/// [`SharedQueryTracker`] aliases. +/// +/// `LmdbStore` is intentionally crate-private, so the store type is sealed: +/// only `FrecencyTracker` / `QueryTracker` can ever instantiate this. +#[allow(private_bounds)] +pub struct SharedDb { + inner: Arc>>, + enabled: bool, +} + +// Hand-written to avoid a spurious `T: Clone` bound — `Arc` is always `Clone`. +impl Clone for SharedDb { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + enabled: self.enabled, + } + } +} + +impl Default for SharedDb { + fn default() -> Self { + Self { + inner: Arc::new(RwLock::new(None)), + enabled: true, + } + } +} + +impl std::fmt::Debug for SharedDb { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SharedDb").field(&T::LABEL).finish() + } +} + +#[allow(private_bounds)] +impl SharedDb { + /// Creates a disabled instance that silently ignores all writes. + pub fn noop() -> Self { + Self { + inner: Arc::new(RwLock::new(None)), + enabled: false, + } + } + + pub fn read(&self) -> Result>, Error> { + self.inner.read().map_err(|_| Error::AcquireFrecencyLock) + } + + pub fn write(&self) -> Result>, Error> { + self.inner.write().map_err(|_| Error::AcquireFrecencyLock) + } + + /// Initialize the store + spawn GC in the background. No-op when disabled. + pub fn init(&self, tracker: T) -> Result<(), Error> { + if !self.enabled { + return Ok(()); + } + + { + let mut guard = self.write()?; + *guard = Some(tracker); + } + + // GC holds a read guard on this lock, so destroy / re-init wait won't race + spawn_lmdb_gc(self.inner.clone()); + Ok(()) + } + + /// Drop the in-memory tracker and delete the on-disk database directory. + /// + /// Acquires the write lock, ensuring all readers (including any active mmap + /// access) are finished before the LMDB environment is closed and the files + /// are removed. + /// + /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no tracker was initialized. + pub fn destroy(&self) -> Result, Error> { + let mut guard = self.write()?; + let Some(tracker) = guard.take() else { + return Ok(None); + }; + let db_path = tracker.env().path().to_path_buf(); + // Drop closes the LMDB env and unmaps the files + drop(tracker); + drop(guard); + std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir { + path: db_path.clone(), + source, + })?; + Ok(Some(db_path)) + } +} + +/// Thread-safe shared handle to the [`FrecencyTracker`] instance. +pub type SharedFrecency = SharedDb; + +/// Thread-safe shared handle to the [`QueryTracker`] instance. +pub type SharedQueryTracker = SharedDb; diff --git a/crates/fff-core/src/simd_path.rs b/crates/fff-core/src/simd_path.rs new file mode 100644 index 0000000..881f109 --- /dev/null +++ b/crates/fff-core/src/simd_path.rs @@ -0,0 +1,612 @@ +use ahash::AHashMap; +use smallvec::SmallVec; +use std::borrow::Cow; + +/// SIMD chunk size in bytes (matches NEON/SSE2 register width). +/// This must stay in sync with neo_frizbee's internal chunk size. +pub(crate) const SIMD_CHUNK_BYTES: usize = 16; + +/// 4 chunks = 64 bytes inline, covers ~85% of paths without heap fallback. +const INLINE_CHUNKS: usize = 4; + +pub(crate) type ChunkIndices = SmallVec<[u32; INLINE_CHUNKS]>; + +#[derive(Clone, Copy)] +pub struct ArenaPtr(pub(crate) *const u8); + +// SAFETY: The arena is a read-only immutable part of file sync +unsafe impl Send for ArenaPtr {} +unsafe impl Sync for ArenaPtr {} + +impl ArenaPtr { + #[inline] + pub fn new(ptr: *const u8) -> Self { + Self(ptr) + } + + #[inline] + pub fn null() -> Self { + Self(std::ptr::null()) + } + + #[inline] + pub fn as_ptr(self) -> *const u8 { + self.0 + } +} + +impl std::fmt::Debug for ArenaPtr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "--arena-raw-pointer-0x({:?})", self.0) + } +} + +#[repr(C, align(16))] +#[derive(Clone, Copy)] +pub(crate) struct SimdChunk(pub(crate) [u8; SIMD_CHUNK_BYTES]); + +impl Default for SimdChunk { + #[inline] + fn default() -> Self { + Self([0u8; SIMD_CHUNK_BYTES]) + } +} + +impl std::fmt::Debug for SimdChunk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Show the actual bytes, trimming trailing zeros for readability + let end = self.0.iter().rposition(|&b| b != 0).map_or(0, |i| i + 1); + write!(f, "SimdChunk({:?})", &self.0[..end]) + } +} + +pub use crate::constants::PATH_BUF_SIZE; + +/// Chunk pointer capacity needed for the longest path the platform allows. +pub(crate) const MAX_PATH_CHUNKS: usize = PATH_BUF_SIZE.div_ceil(SIMD_CHUNK_BYTES); + +/// Indices into a shared `SimdChunk` arena representing a file path. +/// +/// All read methods require an explicit `arena_base` pointer from the owning +/// `ChunkedPathStore`. The struct itself contains no raw pointers to the arena +#[derive(Clone)] +pub(crate) struct ChunkedString { + indices: ChunkIndices, + pub byte_len: u16, + /// Byte offset where the filename begins. 0 for root-level files. + pub filename_offset: u16, +} + +impl ChunkedString { + pub fn empty() -> Self { + Self { + indices: SmallVec::new(), + byte_len: 0, + filename_offset: 0, + } + } + + #[inline] + pub fn new(indices: ChunkIndices, byte_len: u16, filename_offset: u16) -> Self { + Self { + indices, + byte_len, + filename_offset, + } + } + + #[cfg(test)] + pub fn chunk_count(&self) -> usize { + self.indices.len() + } + + #[inline] + pub fn resolve_ptrs<'a>(&self, arena: ArenaPtr, buf: &'a mut [*const u8]) -> &'a [*const u8] { + let count = self.indices.len().min(buf.len()); + let base = arena.as_ptr(); + for (i, &idx) in self.indices[..count].iter().enumerate() { + buf[i] = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) }; + } + &buf[..count] + } + + #[inline] + fn write_slice_to_vec( + indices: &[u32], + base: *const u8, + offset_in_chunk: usize, + len: usize, + vec: &mut Vec, + ) { + let mut written = 0usize; + for (i, &idx) in indices.iter().enumerate() { + let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) }; + let chunk_bytes = unsafe { core::slice::from_raw_parts(src, SIMD_CHUNK_BYTES) }; + let start = if i == 0 { offset_in_chunk } else { 0 }; + let end = SIMD_CHUNK_BYTES.min(start + (len - written)); + vec.extend_from_slice(&chunk_bytes[start..end]); + written += end - start; + } + } + + /// Return the filename portion as a `Cow`. + /// + /// When the filename starts at a chunk boundary and fits in one chunk we + /// borrow directly from the arena (zero-copy). Otherwise we allocate. + /// Filenames are almost always <=16 bytes so the fast path dominates. + #[inline] + pub fn filename_cow<'a>(&self, arena: ArenaPtr) -> Cow<'a, str> { + let fname_offset = self.filename_offset as usize; + let fname_len = self.byte_len as usize - fname_offset; + if fname_len == 0 { + return Cow::Borrowed(""); + } + + let base = arena.as_ptr(); + let start_chunk = fname_offset / SIMD_CHUNK_BYTES; + let offset_in_chunk = fname_offset % SIMD_CHUNK_BYTES; + + if offset_in_chunk == 0 && fname_len <= SIMD_CHUNK_BYTES { + let ptr = unsafe { base.add(self.indices[start_chunk] as usize * SIMD_CHUNK_BYTES) }; + let slice = unsafe { core::slice::from_raw_parts(ptr, fname_len) }; + return Cow::Borrowed(unsafe { core::str::from_utf8_unchecked(slice) }); + } + + let mut out = String::with_capacity(fname_len); + let needed_chunks = chunks_needed(offset_in_chunk + fname_len); + Self::write_slice_to_vec( + &self.indices[start_chunk..start_chunk + needed_chunks], + base, + offset_in_chunk, + fname_len, + unsafe { out.as_mut_vec() }, + ); + Cow::Owned(out) + } + + /// Truncates at `buf.len()` if exceeded -- use `[u8; PATH_BUF_SIZE]` to avoid. + #[inline] + pub fn read_to_buf<'a>(&self, arena: ArenaPtr, buf: &'a mut [u8]) -> &'a str { + let total = (self.byte_len as usize).min(buf.len()); + let usable_chunks = total.div_ceil(SIMD_CHUNK_BYTES); + let chunks_to_copy = usable_chunks.min(self.indices.len()); + let base = arena.as_ptr(); + + for (i, &idx) in self.indices[..chunks_to_copy].iter().enumerate() { + let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) }; + let dst_offset = i * SIMD_CHUNK_BYTES; + let take = SIMD_CHUNK_BYTES.min(total - dst_offset); + + unsafe { + core::ptr::copy_nonoverlapping(src, buf.as_mut_ptr().add(dst_offset), take); + } + } + + unsafe { core::str::from_utf8_unchecked(&buf[..total]) } + } + + #[inline] + pub fn write_dir_to(&self, arena: ArenaPtr, out: &mut String) { + out.clear(); + + let dir_len = self.filename_offset as usize; + out.reserve(dir_len); + let dir_chunks = chunks_needed(dir_len).min(self.indices.len()); + let base = arena.as_ptr(); + let vec = unsafe { out.as_mut_vec() }; + for (i, &idx) in self.indices[..dir_chunks].iter().enumerate() { + let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) }; + let take = SIMD_CHUNK_BYTES.min(dir_len - i * SIMD_CHUNK_BYTES); + vec.extend_from_slice(unsafe { core::slice::from_raw_parts(src, take) }); + } + } + + #[inline] + pub fn write_filename_to(&self, arena: ArenaPtr, out: &mut String) { + out.clear(); + + let fname_offset = self.filename_offset as usize; + let fname_len = self.byte_len as usize - fname_offset; + out.reserve(fname_len); + let start_chunk = fname_offset / SIMD_CHUNK_BYTES; + let offset_in_chunk = fname_offset % SIMD_CHUNK_BYTES; + let needed_chunks = chunks_needed(offset_in_chunk + fname_len); + Self::write_slice_to_vec( + &self.indices[start_chunk..start_chunk + needed_chunks], + arena.as_ptr(), + offset_in_chunk, + fname_len, + unsafe { out.as_mut_vec() }, + ); + } + + #[inline] + pub fn write_to_string(&self, arena: ArenaPtr, out: &mut String) { + out.clear(); + + let total = self.byte_len as usize; + if total == 0 { + return; + } + out.reserve(total); + let base = arena.as_ptr(); + let vec = unsafe { out.as_mut_vec() }; + for (i, &idx) in self.indices.iter().enumerate() { + let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) }; + let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES); + vec.extend_from_slice(unsafe { core::slice::from_raw_parts(src, take) }); + } + } +} + +impl std::fmt::Debug for ChunkedString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChunkedString") + .field("indices", &self.indices.as_slice()) + .field("chunks", &self.indices.len()) + .field("byte_len", &self.byte_len) + .field("filename_offset", &self.filename_offset) + .finish() + } +} + +#[inline] +const fn chunks_needed(byte_len: usize) -> usize { + if byte_len == 0 { + 0 + } else { + byte_len.div_ceil(SIMD_CHUNK_BYTES) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ChunkedPathStore { + arena: Vec, +} + +// SAFETY: arena is immutable after construction. Pointers derived from it are +// only read during scoring (no mutation, no reallocation). +unsafe impl Send for ChunkedPathStore {} +unsafe impl Sync for ChunkedPathStore {} + +impl ChunkedPathStore { + pub fn heap_bytes(&self) -> usize { + self.arena.len() * SIMD_CHUNK_BYTES + } + + #[cfg(test)] + fn unique_chunks(&self) -> usize { + self.arena.len() + } + + #[inline] + pub fn as_arena_ptr(&self) -> ArenaPtr { + ArenaPtr::new(self.arena.as_ptr() as *const u8) + } +} + +/// At runtime the builder should be split out from the store after `finish()`. +#[derive(Clone, Debug)] +pub(crate) struct ChunkedPathStoreBuilder { + arena: Vec, + chunk_dedup: AHashMap<[u8; SIMD_CHUNK_BYTES], u32>, +} + +impl ChunkedPathStoreBuilder { + pub fn new(estimated_files: usize) -> Self { + let est_chunks = estimated_files * INLINE_CHUNKS; // we know that most of repos will fit + // most paths into 64 = 16 * INLINE_CHUNKS + Self { + arena: Vec::with_capacity(est_chunks), + chunk_dedup: AHashMap::with_capacity(est_chunks), + } + } + + pub fn finish(self) -> ChunkedPathStore { + ChunkedPathStore { arena: self.arena } + } + + pub fn as_arena_ptr(&self) -> ArenaPtr { + ArenaPtr::new(self.arena.as_ptr() as *const u8) + } + + /// Like [`add_file_immediate`] but for directory paths where the entire + /// string is the "directory" portion (filename_offset == byte_len). + pub fn add_dir_immediate(&mut self, dir_rel_path: &str) -> ChunkedString { + self.add_file_immediate(dir_rel_path, dir_rel_path.len() as u16) + } + + pub fn add_file_immediate(&mut self, rel_path: &str, filename_offset: u16) -> ChunkedString { + let path_bytes = rel_path.as_bytes(); + let byte_len = rel_path.len(); + let mut indices = ChunkIndices::with_capacity(chunks_needed(byte_len)); + + for chunk in path_bytes.chunks(SIMD_CHUNK_BYTES) { + let mut chunk_bytes = [0u8; SIMD_CHUNK_BYTES]; + chunk_bytes[..chunk.len()].copy_from_slice(chunk); + + let arena_idx = match self.chunk_dedup.get(&chunk_bytes) { + Some(&idx) => idx, + None => { + let idx = self.arena.len() as u32; + self.arena.push(SimdChunk(chunk_bytes)); + self.chunk_dedup.insert(chunk_bytes, idx); + idx + } + }; + + indices.push(arena_idx); + } + + ChunkedString::new(indices, byte_len as u16, filename_offset) + } +} + +#[cfg(test)] +pub(crate) fn build_chunked_path_store_from_strings( + rel_paths: &[String], + files: &[crate::types::FileItem], +) -> (ChunkedPathStore, Vec) { + assert_eq!(rel_paths.len(), files.len()); + let mut builder = ChunkedPathStoreBuilder::new(rel_paths.len()); + let strings: Vec = rel_paths + .iter() + .zip(files.iter()) + .map(|(rel_path, file)| builder.add_file_immediate(rel_path, file.path.filename_offset)) + .collect(); + (builder.finish(), strings) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_file_item(path: &str) -> crate::types::FileItem { + let filename_start = path + .rfind(std::path::is_separator) + .map(|i| i + 1) + .unwrap_or(0) as u16; + crate::types::FileItem::new_raw(filename_start, 0, 0, None, false) + } + + fn build_test_store( + paths: &[&str], + ) -> ( + ChunkedPathStore, + Vec, + Vec, + ) { + let mut files: Vec = + paths.iter().map(|p| make_file_item(p)).collect(); + let path_strings: Vec = paths.iter().map(|p| p.to_string()).collect(); + let (store, strings) = build_chunked_path_store_from_strings(&path_strings, &files); + for (i, file) in files.iter_mut().enumerate() { + file.set_path(strings[i].clone()); + } + (store, strings, files) + } + + #[test] + fn test_chunked_store_empty() { + let (store, strings, _files) = build_test_store(&[]); + assert_eq!(strings.len(), 0); + assert_eq!(store.unique_chunks(), 0); + } + + #[test] + fn test_chunked_store_basic() { + let (store, strings, _files) = + build_test_store(&["src/lib.rs", "src/main.rs", "Cargo.toml"]); + let arena = store.as_arena_ptr(); + + assert_eq!(strings.len(), 3); + assert!(store.unique_chunks() >= 2); + + let mut buf = [0u8; 512]; + assert_eq!( + strings[0].read_to_buf(arena, &mut buf).len(), + "src/lib.rs".len() + ); + assert_eq!( + strings[2].read_to_buf(arena, &mut buf).len(), + "Cargo.toml".len() + ); + } + + #[test] + fn test_chunked_string_full_path() { + let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + let mut buf = [0u8; 512]; + assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx"); + assert_eq!(cs.byte_len, 25); + assert_eq!(cs.filename_offset, 15); + } + + #[test] + fn test_chunked_string_dir_and_filename() { + let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + let mut s = String::new(); + cs.write_dir_to(arena, &mut s); + assert_eq!(s, "src/components/"); + cs.write_filename_to(arena, &mut s); + assert_eq!(s, "Button.tsx"); + } + + #[test] + fn test_chunked_string_root_file() { + let (store, strings, _files) = build_test_store(&["Cargo.toml"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + let mut s = String::new(); + cs.write_dir_to(arena, &mut s); + assert_eq!(s, ""); + cs.write_filename_to(arena, &mut s); + assert_eq!(s, "Cargo.toml"); + let mut buf = [0u8; 512]; + assert_eq!(cs.read_to_buf(arena, &mut buf), "Cargo.toml"); + } + + #[test] + fn test_chunked_string_resolve_ptrs() { + let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + let mut ptrs = [std::ptr::null::(); MAX_PATH_CHUNKS]; + let resolved = cs.resolve_ptrs(arena, &mut ptrs); + assert_eq!(resolved.len(), 2); // 25 bytes = 2 chunks + + // Verify we can read back the bytes + let mut reconstructed = Vec::new(); + for (i, &ptr) in resolved.iter().enumerate() { + let chunk = unsafe { std::slice::from_raw_parts(ptr, SIMD_CHUNK_BYTES) }; + let start = i * SIMD_CHUNK_BYTES; + let take = SIMD_CHUNK_BYTES.min(25 - start); + reconstructed.extend_from_slice(&chunk[..take]); + } + assert_eq!( + std::str::from_utf8(&reconstructed).unwrap(), + "src/components/Button.tsx" + ); + } + + #[test] + fn test_resolve_ptrs_path_exceeding_512_bytes() { + // Regression: a fixed 32-ptr buffer covered only 512 bytes while + // PATH_BUF_SIZE (libc::PATH_MAX) allows longer paths, panicking with + // "index out of bounds: the len is 32 but the index is 32" + let mut path = String::new(); + while path.len() < 600 { + path.push_str("deeply_nested_directory_segment/"); + } + path.push_str("needle_file.rs"); + assert!(path.len() > 512 && path.len() < PATH_BUF_SIZE); + + let (store, strings, _files) = build_test_store(&[path.as_str()]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + assert!(cs.chunk_count() > 32, "path must span more than 32 chunks"); + + let mut ptrs = [std::ptr::null::(); MAX_PATH_CHUNKS]; + let resolved = cs.resolve_ptrs(arena, &mut ptrs); + + // Truncation is not acceptable either: it silently drops the tail of + // the path (including the filename here) from fuzzy matching. + assert_eq!( + resolved.len(), + cs.chunk_count(), + "resolve_ptrs must resolve every chunk of a PATH_MAX-legal path" + ); + + let total = cs.byte_len as usize; + let mut reconstructed = Vec::with_capacity(total); + for (i, &ptr) in resolved.iter().enumerate() { + let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES); + reconstructed.extend_from_slice(unsafe { std::slice::from_raw_parts(ptr, take) }); + } + assert_eq!(std::str::from_utf8(&reconstructed).unwrap(), path); + } + + #[test] + fn test_filename_cow_mid_chunk() { + let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + assert_eq!(cs.filename_offset, 15); + assert_eq!(cs.byte_len, 25); + + let fname = cs.filename_cow(arena); + assert_eq!(&*fname, "Button.tsx"); + } + + #[test] + fn test_filename_cow_chunk_aligned() { + let path = "0123456789abcdef/file.txt"; + let (store, strings, _files) = build_test_store(&[path]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + assert_eq!(cs.filename_offset, 17); + let fname = cs.filename_cow(arena); + assert_eq!(&*fname, "file.txt"); + } + + #[test] + fn test_filename_cow_root_file() { + let (store, strings, _files) = build_test_store(&["Cargo.toml"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + assert_eq!(cs.filename_offset, 0); + let fname = cs.filename_cow(arena); + assert_eq!(&*fname, "Cargo.toml"); + } + + #[test] + fn test_chunked_string_long_path() { + let path = "very/deeply/nested/directory/structure/with/many/levels/file.txt"; + let (store, strings, _files) = build_test_store(&[path]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + + let mut buf = [0u8; 512]; + assert_eq!(cs.read_to_buf(arena, &mut buf), path); + assert!( + cs.chunk_count() <= 6, + "should fit inline in ChunkIndices (INLINE_CHUNKS={})", + INLINE_CHUNKS + ); + } + + #[test] + fn test_chunked_string_clone() { + let (store, strings, _files) = build_test_store(&["src/main.rs"]); + let arena = store.as_arena_ptr(); + let cs = &strings[0]; + let cs2 = cs.clone(); + + let mut buf1 = [0u8; 512]; + let mut buf2 = [0u8; 512]; + assert_eq!( + cs.read_to_buf(arena, &mut buf1), + cs2.read_to_buf(arena, &mut buf2) + ); + } + + #[test] + fn test_chunked_string_full_path_roundtrip() { + let paths = [ + "src/components/Button.tsx", + "src/components/ui/DatePicker.tsx", + "very/deeply/nested/directory/structure/file.txt", + "Cargo.toml", + "a.rs", + ]; + let (store, strings, _files) = build_test_store(&paths); + let arena = store.as_arena_ptr(); + + for (i, expected) in paths.iter().enumerate() { + let mut buf = [0u8; 512]; + let got = strings[i].read_to_buf(arena, &mut buf); + assert_eq!(got, *expected, "full path roundtrip failed for file {i}"); + + let mut ds = String::new(); + let mut fs = String::new(); + strings[i].write_dir_to(arena, &mut ds); + strings[i].write_filename_to(arena, &mut fs); + assert_eq!( + format!("{ds}{fs}"), + *expected, + "dir+fname mismatch for file {i}" + ); + } + } +} diff --git a/crates/fff-core/src/simd_string_utils/case.rs b/crates/fff-core/src/simd_string_utils/case.rs new file mode 100644 index 0000000..fc806f4 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/case.rs @@ -0,0 +1,168 @@ +#[inline] +pub fn ascii_swap_case(b: u8) -> u8 { + b ^ 0x20 +} + +#[inline] +fn eq_lowered_scalar(h: *const u8, needle_lower: &[u8]) -> bool { + for (i, &n) in needle_lower.iter().enumerate() { + if unsafe { *h.add(i) }.to_ascii_lowercase() != n { + return false; + } + } + true +} + +/// AVX2 only has a **signed** byte compare (`cmpgt`), but we need an +/// **unsigned** range check (`'A' <= byte <= 'Z'`). XOR-ing every byte with +/// `0x80` maps the unsigned range `[0, 255]` into the signed range +/// `[-128, 127]` preserving order, so signed `cmpgt` becomes correct +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn eq_lowered_avx2(h: *const u8, needle_lower: &[u8]) -> bool { + use core::arch::x86_64::*; + + let len = needle_lower.len(); + let mut i = 0usize; + + let flip = _mm256_set1_epi8(0x80u8 as i8); + let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8); + let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8); + let bit20 = _mm256_set1_epi8(0x20u8 as i8); + + while i + 32 <= len { + let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) }; + let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) }; + + // Signed-domain range check selects uppercase lanes, OR bit 5 folds them. + let x = _mm256_xor_si256(hv, flip); + let ge_a = _mm256_cmpgt_epi8(x, a_minus_1); + let le_z = _mm256_cmpgt_epi8(z_plus_1, x); + let upper = _mm256_and_si256(ge_a, le_z); + let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20)); + + let eq = _mm256_cmpeq_epi8(folded, nv); + if _mm256_movemask_epi8(eq) != -1i32 { + return false; + } + + i += 32; + } + + while i < len { + if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] { + return false; + } + i += 1; + } + true +} + +/// Unsigned range checks (`vcge`/`vcle`) detect uppercase ASCII, bit 5 folds +/// to lowercase, then equality is checked via udot: xors the folded haystack +/// with the pre-lowered needle and dot-product the difference with itself +/// any non-zero byte produces a non-zero u32 lane. udot is emitted via inline +/// asm because `vdotq_u32` is still behind an unstable feature gate. +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon,dotprod")] +unsafe fn eq_lowered_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool { + use core::arch::aarch64::*; + + let len = needle_lower.len(); + let mut i = 0usize; + + let a_val = vdupq_n_u8(b'A'); + let z_val = vdupq_n_u8(b'Z'); + let bit20 = vdupq_n_u8(0x20); + + while i + 16 <= len { + let hv = unsafe { vld1q_u8(h.add(i)) }; + let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) }; + + let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val)); + let folded = vorrq_u8(hv, vandq_u8(upper, bit20)); + let xored = veorq_u8(folded, nv); + + let dots: uint32x4_t; + let zero = vdupq_n_u32(0); + unsafe { + core::arch::asm!( + "udot {d:v}.4s, {a:v}.16b, {b:v}.16b", + d = inlateout(vreg) zero => dots, + a = in(vreg) xored, + b = in(vreg) xored, + ); + } + + if vmaxvq_u32(dots) != 0 { + return false; + } + + i += 16; + } + + while i < len { + if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] { + return false; + } + i += 1; + } + true +} + +/// Case-insensitive equality of `needle_lower` against the haystack bytes +/// starting at `h`. `needle_lower` must be pre-lowercased (ASCII). +/// +/// # Safety +/// `h` must be valid for reads of `needle_lower.len()` bytes. +#[inline] +pub(crate) unsafe fn eq_lowered_case(haystack: *const u8, needle_lower: &[u8]) -> bool { + #[cfg(target_arch = "x86_64")] + { + if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") { + return unsafe { eq_lowered_avx2(haystack, needle_lower) }; + } + } + #[cfg(target_arch = "aarch64")] + { + if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") { + return unsafe { eq_lowered_neon_dotprod(haystack, needle_lower) }; + } + } + + eq_lowered_scalar(haystack, needle_lower) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn eq_lowered(haystack: &[u8], needle_lower: &[u8]) -> bool { + assert!(haystack.len() >= needle_lower.len()); + unsafe { eq_lowered_case(haystack.as_ptr(), needle_lower) } + } + + #[test] + fn swap_case_toggles_letters() { + assert_eq!(ascii_swap_case(b'n'), b'N'); + assert_eq!(ascii_swap_case(b'N'), b'n'); + assert_eq!(ascii_swap_case(b'z'), b'Z'); + } + + #[test] + fn eq_matches_std_semantics() { + assert!(eq_lowered(b"Hello", b"hello")); + assert!(eq_lowered(b"HELLO WORLD", b"hello")); + assert!(!eq_lowered(b"Hellp", b"hello")); + // Non-letters must not fold: '[' (0x5B) vs '{' (0x7B) differ only in bit 5. + assert!(!eq_lowered(b"A[", b"a{")); + assert!(eq_lowered(b"A{", b"a{")); + // Long inputs exercise the SIMD kernels. + let hay = b"INT STRUCT MUTEX *LOCK(STRUCT MUTEX *LOCK) { RETURN 0; }"; + let needle: Vec = hay.iter().map(|b| b.to_ascii_lowercase()).collect(); + assert!(eq_lowered(hay, &needle)); + let mut bad = needle.clone(); + *bad.last_mut().unwrap() = b'!'; + assert!(!eq_lowered(hay, &bad)); + } +} diff --git a/crates/fff-core/src/simd_string_utils/memmem.rs b/crates/fff-core/src/simd_string_utils/memmem.rs new file mode 100644 index 0000000..a5bda60 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/memmem.rs @@ -0,0 +1,494 @@ +use super::case::{ascii_swap_case, eq_lowered_case}; +use smallvec::SmallVec; + +// Byte frequency table stolen from memchr +const BYTE_FREQUENCIES: [u8; 256] = [ + 55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00 + 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10 + 255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20 + 208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30 + 120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185, + 167, // 0x40 A-O + 186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114, + 223, // 0x50 P-_ + 151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246, + 244, // 0x60 a-o + 231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127, + 27, // 0x70 p-DEL + 212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80 + 207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90 + 118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0 + 166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0 +]; + +#[inline] +fn rank(lower: u8) -> u8 { + if lower.is_ascii_lowercase() { + BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[ascii_swap_case(lower) as usize]) + } else { + BYTE_FREQUENCIES[lower as usize] + } +} + +/// Pick two needle positions with the rarest bytes (case-insensitive) +fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) { + debug_assert!(needle_lower.len() >= 2); + + let mut best1 = (u8::MAX, 0usize); // (rank, position) + let mut best2 = (u8::MAX, 1usize); + + for (i, &b) in needle_lower.iter().enumerate() { + let r = rank(b); + if r < best1.0 { + best2 = best1; + best1 = (r, i); + } else if r < best2.0 && i != best1.1 { + best2 = (r, i); + } + } + + let i1 = best1.1.min(best2.1); + let i2 = best1.1.max(best2.1); + (i1, i2) +} + +/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF) +/// Bit *i* of the result corresponds to byte *i* of the input vector +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[inline] +unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 { + use core::arch::aarch64::*; + + // AND each byte with its bit-position mask, then horizontally sum each half. + static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128]; + let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) }; + let masked = vandq_u8(v, bit_mask); + let lo = vaddv_u8(vget_low_u8(masked)); + let hi = vaddv_u8(vget_high_u8(masked)); + (lo as u16) | ((hi as u16) << 8) +} + +/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration, +/// checking two rare bytes (case-insensitive) simultaneously +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn find_packed_pair_avx2( + haystack: &[u8], + needle_lower: &[u8], + i1: usize, + i2: usize, +) -> Option { + use core::arch::x86_64::*; + + let n = needle_lower.len(); + let hlen = haystack.len(); + let ptr = haystack.as_ptr(); + let last_start = hlen - n; // last valid match-start position + + let b1 = needle_lower[i1]; + let b1_alt = if b1.is_ascii_lowercase() { + ascii_swap_case(b1) + } else { + b1 + }; + let b2 = needle_lower[i2]; + let b2_alt = if b2.is_ascii_lowercase() { + ascii_swap_case(b2) + } else { + b2 + }; + + let v1_lo = _mm256_set1_epi8(b1 as i8); + let v1_hi = _mm256_set1_epi8(b1_alt as i8); + let v2_lo = _mm256_set1_epi8(b2 as i8); + let v2_hi = _mm256_set1_epi8(b2_alt as i8); + + // Loads come from ptr+offset+i1 and ptr+offset+i2, so we need offset + max(i1,i2) + 31 < hlen. + let max_idx = i1.max(i2); + let max_offset = hlen.saturating_sub(max_idx + 32); + let mut offset = 0usize; + + while offset <= max_offset { + let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) }; + let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) }; + + // Case-insensitive match: OR both case variants, then AND the two positions. + let eq1 = _mm256_or_si256( + _mm256_cmpeq_epi8(chunk1, v1_lo), + _mm256_cmpeq_epi8(chunk1, v1_hi), + ); + let eq2 = _mm256_or_si256( + _mm256_cmpeq_epi8(chunk2, v2_lo), + _mm256_cmpeq_epi8(chunk2, v2_hi), + ); + + let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32; + + // Candidates are visited in increasing position order, so the first + // verified candidate is the leftmost match + while mask != 0 { + let bit = mask.trailing_zeros() as usize; + let candidate = offset + bit; + if candidate > last_start { + return None; + } + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + mask &= mask - 1; + } + + offset += 32; + } + + // handle remaining characters + if offset <= last_start { + let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) { + i1 + } else { + i2 + }; + let rare_byte = needle_lower[rare_pos]; + let tail_start = offset + rare_pos; + let tail_end = last_start + rare_pos + 1; + if tail_start < tail_end { + let tail_space = &haystack[tail_start..tail_end]; + if rare_byte.is_ascii_lowercase() { + for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } else { + for pos in memchr::memchr_iter(rare_byte, tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } + } + } + + None +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn find_packed_pair_neon( + haystack: &[u8], + needle_lower: &[u8], + i1: usize, + i2: usize, +) -> Option { + use core::arch::aarch64::*; + + let n = needle_lower.len(); + let hlen = haystack.len(); + let ptr = haystack.as_ptr(); + let last_start = hlen - n; + + let b1 = needle_lower[i1]; + let b1_alt = if b1.is_ascii_lowercase() { + ascii_swap_case(b1) + } else { + b1 + }; + let b2 = needle_lower[i2]; + let b2_alt = if b2.is_ascii_lowercase() { + ascii_swap_case(b2) + } else { + b2 + }; + + let v1_lo = vdupq_n_u8(b1); + let v1_hi = vdupq_n_u8(b1_alt); + let v2_lo = vdupq_n_u8(b2); + let v2_hi = vdupq_n_u8(b2_alt); + + let max_idx = i1.max(i2); + let max_offset = hlen.saturating_sub(max_idx + 16); + let mut offset = 0usize; + + while offset <= max_offset { + let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) }; + let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) }; + + // Case-insensitive match: OR both case variants, then AND the two positions. + let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi)); + let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi)); + + let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) }; + + while mask != 0 { + let bit = mask.trailing_zeros() as usize; + let candidate = offset + bit; + if candidate > last_start { + return None; + } + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + mask &= mask - 1; + } + + offset += 16; + } + + // Tail: remaining positions that couldn't fill a full vector. + if offset <= last_start { + let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) { + i1 + } else { + i2 + }; + let rare_byte = needle_lower[rare_pos]; + let tail_start = offset + rare_pos; + let tail_end = last_start + rare_pos + 1; + if tail_start < tail_end { + let tail_space = &haystack[tail_start..tail_end]; + if rare_byte.is_ascii_lowercase() { + for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } else { + for pos in memchr::memchr_iter(rare_byte, tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } + } + } + + None +} + +fn find_first_byte_with_memchr(haystack: &[u8], needle_lower: &[u8]) -> Option { + let n = needle_lower.len(); + debug_assert!(n >= 1 && n <= haystack.len()); + + let search_space = &haystack[..=haystack.len() - n]; + let first = needle_lower[0]; + + if first.is_ascii_lowercase() { + let alt = ascii_swap_case(first); + for pos in memchr::memchr2_iter(first, alt, search_space) { + if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } { + return Some(pos); + } + } + } else { + for pos in memchr::memchr_iter(first, search_space) { + if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } { + return Some(pos); + } + } + } + None +} + +/// ASCII case-insensitive substring search returning the leftmost match +/// position. `needle_lower` must be pre-lowercased (ASCII). +// pub because it is used in out of the crate benchmarks +#[doc(hidden)] // it's pub only for benches +pub fn find(haystack: &[u8], needle_lower: &[u8]) -> Option { + let n = needle_lower.len(); + if n == 0 { + return Some(0); + } + if n > haystack.len() { + return None; + } + + if n == 1 { + let first = needle_lower[0]; + return if first.is_ascii_lowercase() { + memchr::memchr2(first, ascii_swap_case(first), haystack) + } else { + memchr::memchr(first, haystack) + }; + } + + #[cfg_attr( + not(any(target_arch = "x86_64", target_arch = "aarch64")), + allow(unused_variables) + )] + let (i1, i2) = select_rare_pair(needle_lower); + + #[cfg(target_arch = "x86_64")] + { + if std::is_x86_feature_detected!("avx2") { + // Need enough haystack for at least one vector load. + let max_idx = i1.max(i2); + if haystack.len() >= max_idx + 32 { + return unsafe { find_packed_pair_avx2(haystack, needle_lower, i1, i2) }; + } + } + } + + #[cfg(target_arch = "aarch64")] + { + // Packed-pair wins when the first byte is common (memchr2 drowns in + // false positives), but a rare first byte (z, q, x, ...) makes + // memchr2's raw throughput dominate. Threshold 200 on the frequency + // table splits common letters (s=243, e=253) from rare ones (z=152). + let first_byte_rank = rank(needle_lower[0]); + let max_idx = i1.max(i2); + if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 { + return unsafe { find_packed_pair_neon(haystack, needle_lower, i1, i2) }; + } + } + + // fallbacks to memchr based implementation cause we still have it and it supports more SIMD backends + // TODO convert all the supported backend by memchr and get rid of the fallback + find_first_byte_with_memchr(haystack, needle_lower) +} + +/// A case insensitive find that works better with smaller strings, doesn't unwrap a complicated +/// AVX backend we use for grep because only cpu flags check takes usually more time than find itself +pub fn find_case_insensitive_short(haystack: &[u8], needle: &[u8]) -> Option { + debug_assert!(haystack.len() < 1024); + let mut needle_lower: SmallVec<[u8; 64]> = SmallVec::from_slice(needle); + needle_lower.make_ascii_lowercase(); + + find(haystack, &needle_lower) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference_find(haystack: &[u8], needle_lower: &[u8]) -> Option { + if needle_lower.is_empty() { + return Some(0); + } + if needle_lower.len() > haystack.len() { + return None; + } + haystack + .windows(needle_lower.len()) + .position(|w| w.eq_ignore_ascii_case(needle_lower)) + } + + #[test] + fn basic_case_insensitive() { + assert_eq!(find(b"Hello World", b"hello"), Some(0)); + assert_eq!(find(b"Hello World", b"world"), Some(6)); + assert_eq!(find(b"NOMORE bugs", b"nomore"), Some(0)); + assert_eq!(find(b"Hello World", b"xyz"), None); + assert!(find(b"Hello World", b"o w").is_some()); + } + + #[test] + fn edge_cases() { + assert_eq!(find(b"ab", b"ab"), Some(0)); + assert_eq!(find(b"AB", b"ab"), Some(0)); + assert_eq!(find(b"a", b"ab"), None); + assert_eq!(find(b"anything", b""), Some(0)); + assert_eq!(find(b"", b"x"), None); + assert_eq!(find(b"xxA", b"a"), Some(2)); + assert_eq!(find(b"xx:", b":"), Some(2)); + } + + #[test] + fn returns_leftmost_match() { + assert_eq!(find(b"foo FOO foo", b"foo"), Some(0)); + let mut big = vec![b'.'; 300]; + big[100..103].copy_from_slice(b"FoO"); + big[200..203].copy_from_slice(b"foo"); + assert_eq!(find(&big, b"foo"), Some(100)); + } + + #[test] + fn non_letter_bytes_do_not_case_fold() { + // '[' (0x5B) and '{' (0x7B) differ only in bit 5 but are not letters. + // A fold implemented as a bare `| 0x20` would falsely match these. + assert_eq!(find(b"A[", b"a{"), None); + assert_eq!(find(b"x@y", b"x`y"), None); + assert_eq!(find(b"a]b", b"a}b"), None); + assert_eq!(find(b"A{", b"a{"), Some(0)); + } + + #[test] + fn matches_reference_on_random_inputs() { + // Deterministic xorshift PRNG — no external deps. + let mut state = 0x9E3779B97F4A7C15u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + + // Alphabet with letters, both-case pairs, and 0x20-differing symbols. + let alphabet = b"aAbBzZ [{@`]}^~_0.\n"; + for _ in 0..2000 { + let hlen = (next() % 200) as usize; + let nlen = (next() % 8) as usize; + let haystack: Vec = (0..hlen) + .map(|_| alphabet[(next() % alphabet.len() as u64) as usize]) + .collect(); + let needle: Vec = (0..nlen) + .map(|_| alphabet[(next() % alphabet.len() as u64) as usize].to_ascii_lowercase()) + .collect(); + + assert_eq!( + find(&haystack, &needle), + reference_find(&haystack, &needle), + "mismatch for haystack={:?} needle={:?}", + haystack, + needle, + ); + } + } + + #[test] + fn long_haystack_simd_paths() { + let haystack = + b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb"; + assert_eq!(find(haystack, b"this_is_a_long_needle_test"), Some(32)); + assert_eq!(find(haystack, b"this_is_a_long_needle_testz"), None); + + // Needle >= 16 bytes exercises SIMD verify. + let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }"; + assert_eq!(find(haystack2, b"struct mutex *lock"), Some(4)); + + let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + assert_eq!(find(upper_hay, b"qrstuvwxyz0123456789a"), Some(16)); + assert_eq!(find(upper_hay, b"qrstuvwxyz01234567899"), None); + + // Needle at very end / very start. + let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me"; + assert_eq!(find(end_hay, b"find_me"), Some(end_hay.len() - 7)); + assert_eq!(find(end_hay, b"xx"), Some(0)); + + // 1KB haystack with needle near the end. + let mut big = vec![b'z'; 1024]; + big[1000..1010].copy_from_slice(b"hElLo_WoRl"); + assert_eq!(find(&big, b"hello_wo"), Some(1000)); + assert_eq!(find(&big, b"hello_world"), None); + } + + #[test] + fn rare_pair_selection() { + let (i1, i2) = select_rare_pair(b"nomore"); + let ranks: Vec = b"nomore".iter().map(|&b| rank(b)).collect(); + let (r1, r2) = (ranks[i1], ranks[i2]); + for (i, &r) in ranks.iter().enumerate() { + if i != i1 && i != i2 { + assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal"); + } + } + } +} diff --git a/crates/fff-core/src/simd_string_utils/mod.rs b/crates/fff-core/src/simd_string_utils/mod.rs new file mode 100644 index 0000000..bb66b43 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/mod.rs @@ -0,0 +1,5 @@ +//! SIMD-accelerated string utilities: case flipping/folding and +//! case-insensitive substring search. + +pub mod case; +pub mod memmem; diff --git a/crates/fff-core/src/sort_buffer.rs b/crates/fff-core/src/sort_buffer.rs new file mode 100644 index 0000000..940169b --- /dev/null +++ b/crates/fff-core/src/sort_buffer.rs @@ -0,0 +1,131 @@ +use parking_lot::Mutex; +use std::mem::MaybeUninit; + +// this originally happen to be in TLS but there is a limit of TLS +// + the storage itself is not free, so now we rely on the fact that most calls +// are sequential in practice and allocate ONLY when we have a parallel access +static SORT_BUFFER: Mutex> = Mutex::new(Vec::new()); + +fn ensure_capacity(buf: &mut Vec, required: usize) { + if buf.capacity() < required { + let len = buf.len(); + buf.reserve(required - len); + } +} + +struct SharedSortBuf { + guard: parking_lot::MutexGuard<'static, Vec>, +} + +impl SharedSortBuf { + fn as_slice_mut(&mut self, len: usize) -> &mut [MaybeUninit] { + let align = std::mem::align_of::>(); + let size = std::mem::size_of::>(); + let required = len.saturating_mul(size).saturating_add(align); + ensure_capacity(&mut self.guard, required); + + // SAFETY: the Vec is only 1-byte aligned, so we over-allocate by + // `align` bytes and shift the pointer to satisfy T's alignment. + // Callers never read uninitialised data through the returned slice. + unsafe { + let ptr = self.guard.as_mut_ptr(); + let offset = ptr.align_offset(align); + debug_assert!(offset != usize::MAX && offset + len * size <= self.guard.capacity()); + std::slice::from_raw_parts_mut(ptr.add(offset) as *mut MaybeUninit, len) + } + } +} + +fn try_lock_shared_buf() -> Option { + SORT_BUFFER.try_lock().map(|guard| SharedSortBuf { guard }) +} + +pub fn sort_with_buffer(slice: &mut [T], compare: F) +where + F: FnMut(&T, &T) -> std::cmp::Ordering, +{ + match try_lock_shared_buf() { + Some(mut buf) => { + let typed = buf.as_slice_mut::(slice.len()); + glidesort::sort_with_buffer_by(slice, typed, compare); + } + None => glidesort::sort_by(slice, compare), + } +} + +pub fn sort_by_key_with_buffer(slice: &mut [T], key_fn: F) +where + K: Ord, + F: FnMut(&T) -> K, +{ + match try_lock_shared_buf() { + Some(mut buf) => { + let typed = buf.as_slice_mut::(slice.len()); + glidesort::sort_with_buffer_by_key(slice, typed, key_fn); + } + None => glidesort::sort_by_key(slice, key_fn), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sort_with_buffer() { + let mut data = vec![5, 2, 8, 1, 9]; + sort_with_buffer(&mut data, |a, b| a.cmp(b)); + assert_eq!(data, vec![1, 2, 5, 8, 9]); + } + + #[test] + fn test_sort_by_key_with_buffer() { + let mut data = vec![(1, 50), (2, 20), (3, 80), (4, 10), (5, 90)]; + sort_by_key_with_buffer(&mut data, |a| a.1); + assert_eq!(data, vec![(4, 10), (2, 20), (1, 50), (3, 80), (5, 90)]); + } + + #[test] + fn test_reverse_sort() { + let mut data = vec![1, 2, 3, 4, 5]; + sort_with_buffer(&mut data, |a, b| b.cmp(a)); + assert_eq!(data, vec![5, 4, 3, 2, 1]); + } + + #[test] + fn test_empty_slice() { + let mut data: Vec = vec![]; + sort_with_buffer(&mut data, |a, b| a.cmp(b)); + assert_eq!(data, Vec::::new()); + } + + #[test] + fn test_single_element() { + let mut data = vec![42]; + sort_with_buffer(&mut data, |a, b| a.cmp(b)); + assert_eq!(data, vec![42]); + } + + #[test] + fn test_with_duplicates() { + let mut data = vec![3, 1, 4, 1, 5, 9, 2, 6, 5]; + sort_with_buffer(&mut data, |a, b| a.cmp(b)); + assert_eq!(data, vec![1, 1, 2, 3, 4, 5, 5, 6, 9]); + } + + #[test] + fn test_descending_order() { + let mut data = vec![3, 1, 4, 1, 5, 9, 2, 6, 5]; + sort_with_buffer(&mut data, |a, b| b.cmp(a)); + assert_eq!(data, vec![9, 6, 5, 5, 4, 3, 2, 1, 1]); + } + + #[test] + fn test_simple_descending() { + let mut data = vec![100, 300, 200]; + sort_with_buffer(&mut data, |a, b| b.cmp(a)); + assert_eq!(data[0], 300); + assert_eq!(data[1], 200); + assert_eq!(data[2], 100); + } +} diff --git a/crates/fff-core/src/stable_vec.rs b/crates/fff-core/src/stable_vec.rs new file mode 100644 index 0000000..950607a --- /dev/null +++ b/crates/fff-core/src/stable_vec.rs @@ -0,0 +1,177 @@ +use std::alloc::{self, Layout}; +use std::ptr::NonNull; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Vector that guarantees no re-alloc happening at runtime +pub(crate) struct StableVec { + inner: Arc>, +} + +struct StableBuf { + ptr: NonNull, + cap: usize, + /// Atomic because: + /// 1. `push(&self)` must mutate this through a shared `&StableBuf`, + /// which requires interior mutability. + /// 2. Arc clones (e.g. post-scan snapshots) read `len` outside the + /// picker lock, concurrent with an appending writer. Acquire/Release + /// on len is what makes "observed len ⇒ element bytes initialized" + /// actually hold. + /// + /// Arc wrapping only shares ownership of the buffer; it does NOT + /// synchronize access to fields inside the shared buffer. + len: AtomicUsize, +} + +// SAFETY: StableBuf is a thread-safe container when T is send + sync +// There is another application level constraint: mutations are safe +// when they are atomic updates, not read + update. +unsafe impl Send for StableBuf {} +unsafe impl Sync for StableBuf {} + +impl Drop for StableBuf { + fn drop(&mut self) { + let len = *self.len.get_mut(); + unsafe { + std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), len)); + if self.cap > 0 { + let layout = Layout::array::(self.cap).expect("layout"); + alloc::dealloc(self.ptr.as_ptr().cast(), layout); + } + } + } +} + +impl StableVec { + pub fn from_vec_with_reserve(mut vec: Vec, extra: usize) -> Self { + vec.reserve(extra); + let cap = vec.capacity(); + let len = vec.len(); + + let inner = if cap == 0 { + StableBuf { + ptr: NonNull::dangling(), + cap: 0, + len: AtomicUsize::new(0), + } + } else { + // Take ownership of the Vec's buffer without running element + // drops; we hand them off to the StableBuf. + let mut vec = std::mem::ManuallyDrop::new(vec); + let ptr = NonNull::new(vec.as_mut_ptr()).expect("non-null"); + StableBuf { + ptr, + cap, + len: AtomicUsize::new(len), + } + }; + + Self { + inner: Arc::new(inner), + } + } + + /// Append. Returns `false` if capacity is exhausted (item dropped). + /// + /// Safe to call via `&self` as long as the caller holds the outer + /// picker write lock (single-writer invariant). + #[inline] + pub fn push(&self, item: T) -> bool { + let cap = self.inner.cap; + let len = self.inner.len.load(Ordering::Acquire); + if len >= cap { + debug_assert!( + false, + "StableVec: push would exceed capacity ({len} at capacity {cap})" + ); + tracing::error!( + len, + capacity = cap, + "StableVec: capacity exhausted — dropping item to prevent reallocation" + ); + return false; + } + unsafe { + std::ptr::write(self.inner.ptr.as_ptr().add(len), item); + } + self.inner.len.store(len + 1, Ordering::Release); + true + } + + // this method is specifically private because you probably need to use + // live_count if you are trying to access this method + #[inline] + pub fn len(&self) -> usize { + self.inner.len.load(Ordering::Acquire) + } + + /// Mutable element access for in-place field updates. Never shifts. + /// + /// LATENT UB: produces `&mut T` aliasing Arc-shared memory; the + /// `&mut self` on StableVec does NOT imply unique access to the + /// `StableBuf` when sibling Arc clones exist. Safe in practice + /// because callers hold the picker write lock and writes target + /// disjoint fields, but strictly forbidden by the aliasing model. + #[inline] + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + let len = self.inner.len.load(Ordering::Acquire); + if index >= len { + return None; + } + unsafe { Some(&mut *self.inner.ptr.as_ptr().add(index)) } + } + + #[inline] + pub fn last(&self) -> Option<&T> { + let len = self.len(); + if len == 0 { + None + } else { + unsafe { Some(&*self.inner.ptr.as_ptr().add(len - 1)) } + } + } + + /// Iterate mutably for in-place field updates. Never shifts storage. + /// Same latent-UB caveat as [`get_mut`]: `&mut T` into Arc-shared memory. + #[inline] + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> { + let len = self.inner.len.load(Ordering::Acquire); + unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len).iter_mut() } + } +} + +impl Clone for StableVec { + #[inline] + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl std::fmt::Debug for StableVec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("StableVec").field(&self.len()).finish() + } +} + +impl std::ops::Deref for StableVec { + type Target = [T]; + #[inline] + fn deref(&self) -> &[T] { + let len = self.len(); + unsafe { std::slice::from_raw_parts(self.inner.ptr.as_ptr(), len) } + } +} + +impl std::ops::DerefMut for StableVec { + /// LATENT UB: `&mut [T]` aliases Arc-shared memory. Kept for + /// Index/IndexMut ergonomics at call sites that write disjoint + /// fields under the picker write lock. See module-level doc. + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + let len = self.inner.len.load(Ordering::Acquire); + unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len) } + } +} diff --git a/crates/fff-core/src/types.rs b/crates/fff-core/src/types.rs new file mode 100644 index 0000000..932fb10 --- /dev/null +++ b/crates/fff-core/src/types.rs @@ -0,0 +1,971 @@ +use std::io::Read; +use std::path::{Path, PathBuf}; +#[cfg(not(target_os = "windows"))] +use std::sync::OnceLock; +use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering}; + +#[cfg(not(target_os = "windows"))] +use crate::constants::{FRESH_MMAP_THRESHOLD, MMAP_THRESHOLD}; +use crate::constants::{MAX_CACHED_CONTENT_BYTES, MAX_FFFILE_SIZE, PATH_BUF_SIZE}; +use crate::constraints::Constrainable; +use crate::query_tracker::QueryMatchEntry; +use crate::simd_path::ArenaPtr; +use fff_query_parser::{FFFQuery, FuzzyQuery, Location}; + +/// Different sources of the string storage used by FFF +/// implements as a deduplicated 16-bytes alined heap +/// can be stored in RAM or on disk +pub trait FFFStringStorage { + /// Resolve the arena for a [`FileItem`] (handles base vs overflow split). + fn arena_for(&self, file: &FileItem) -> ArenaPtr; + + /// The base arena (scan-time paths). + fn base_arena(&self) -> ArenaPtr; + /// The overflow arena (paths added after the last full scan). + fn overflow_arena(&self) -> ArenaPtr; +} + +impl FFFStringStorage for ArenaPtr { + #[inline] + fn arena_for(&self, _file: &FileItem) -> ArenaPtr { + *self + } + + #[inline] + fn base_arena(&self) -> ArenaPtr { + *self + } + + #[inline] + fn overflow_arena(&self) -> ArenaPtr { + *self + } +} + +pub trait FileSliceExt { + fn live_count(&self) -> usize; +} + +impl FileSliceExt for [FileItem] { + #[inline] + fn live_count(&self) -> usize { + self.iter().filter(|f| !f.is_deleted()).count() + } +} + +pub struct FileItemFlags; + +impl FileItemFlags { + pub const BINARY: u8 = 1 << 0; + /// Tombstone — file was deleted but index slot is preserved so + /// bigram indices for other files stay valid. + pub const DELETED: u8 = 1 << 1; + /// File was added after the last full reindex; its indices point + /// into the overflow builder arena, not the base arena. + pub const OVERFLOW: u8 = 1 << 2; +} + +pub struct DirFlags; + +impl DirFlags { + pub const OVERFLOW: u8 = 1 << 0; +} + +/// A directory in the file index. Shares chunk arena with file paths. +#[derive(Debug)] +pub struct DirItem { + flags: u8, + pub(crate) path: crate::simd_path::ChunkedString, + /// Byte offset where the last path segment begins (e.g. for `src/components/` + /// this is 4, pointing to `components/`). Used for dirname-bonus scoring. + last_segment_offset: u16, + /// Maximum `access_frecency_score` among direct child files. + /// Atomic so parallel frecency updates can write directly without juggling. + max_access_frecency: AtomicI32, +} + +impl Clone for DirItem { + fn clone(&self) -> Self { + Self { + flags: self.flags, + path: self.path.clone(), + last_segment_offset: self.last_segment_offset, + max_access_frecency: AtomicI32::new(self.max_access_frecency()), + } + } +} + +impl DirItem { + #[inline(always)] + pub fn is_overflow(&self) -> bool { + self.flags & DirFlags::OVERFLOW != 0 + } + + pub(crate) fn new(path: crate::simd_path::ChunkedString, last_segment_offset: u16) -> Self { + Self { + path, + flags: 0, + last_segment_offset, + max_access_frecency: AtomicI32::new(0), + } + } + + /// Byte offset of the last path segment within the directory path. + #[inline] + pub fn last_segment_offset(&self) -> u16 { + self.last_segment_offset + } + + /// Current max access frecency score. + #[inline] + pub fn max_access_frecency(&self) -> i32 { + self.max_access_frecency.load(Ordering::Relaxed) + } + + /// Atomically update the directory's frecency score if the given score is larger. + /// Safe to call from parallel threads. + #[inline] + pub fn update_frecency_if_larger(&self, score: i32) { + self.max_access_frecency.fetch_max(score, Ordering::Relaxed); + } + + /// Reset frecency to zero (used before full recomputation). + #[inline] + pub fn reset_frecency(&self) { + self.max_access_frecency.store(0, Ordering::Relaxed); + } + + pub(crate) fn read_relative_path<'a>(&self, arena: ArenaPtr, buf: &'a mut [u8]) -> &'a str { + self.path.read_to_buf(arena, buf) + } + + /// Relative dir path as owned String (cold path). + pub fn relative_path(&self, arena: impl FFFStringStorage) -> String { + let mut out = String::new(); + let ptr = if self.is_overflow() { + arena.overflow_arena() + } else { + arena.base_arena() + }; + + self.path.write_to_string(ptr, &mut out); + out + } + + /// Write the last segment (dirname) of this directory path to `out`. + pub fn write_dir_name(&self, arena: ArenaPtr, out: &mut String) { + out.clear(); + let total = self.path.byte_len as usize; + let offset = self.last_segment_offset as usize; + if offset >= total { + return; + } + // Read the full path, then slice from last_segment_offset + let mut buf = [0u8; PATH_BUF_SIZE]; + let full = self.path.read_to_buf(arena, &mut buf); + out.push_str(&full[offset..]); + } + + /// The dirname (last segment) as an owned String. Cold path. + pub fn dir_name(&self, arena: impl FFFStringStorage) -> String { + let mut out = String::new(); + let ptr = if self.is_overflow() { + arena.overflow_arena() + } else { + arena.base_arena() + }; + self.write_dir_name(ptr, &mut out); + out + } + + /// A path = base_path + "/" + relative. Cold path, allocates. + pub fn absolute_path(&self, arena: impl FFFStringStorage, base_path: &Path) -> PathBuf { + let rel = self.relative_path(arena); + if rel.is_empty() { + base_path.to_path_buf() + } else { + base_path.join(&rel) + } + } +} + +impl Constrainable for DirItem { + #[inline] + fn write_file_name(&self, arena: ArenaPtr, out: &mut String) { + // For dirs, the "file name" equivalent is the last path segment + self.write_dir_name(arena, out); + } + + #[inline] + fn write_relative_path(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_to_string(arena, out); + } + + #[inline] + fn git_status(&self) -> Option { + None + } + + #[inline] + fn is_overflow(&self) -> bool { + DirItem::is_overflow(self) + } +} + +#[derive(Debug)] +pub struct FileItem { + pub size: u64, + pub modified: u64, + pub access_frecency_score: i16, + pub modification_frecency_score: i16, + pub git_status: Option, + pub(crate) path: crate::simd_path::ChunkedString, + pub(crate) parent_dir_index: u32, + flags: AtomicU8, + /// Lazy mmap cache. Only populated by the actual file read, controlled by the budget. + #[cfg(not(target_os = "windows"))] + content: OnceLock, +} + +impl Clone for FileItem { + fn clone(&self) -> Self { + Self { + path: self.path.clone(), + parent_dir_index: self.parent_dir_index, + size: self.size, + modified: self.modified, + access_frecency_score: self.access_frecency_score, + modification_frecency_score: self.modification_frecency_score, + git_status: self.git_status, + flags: AtomicU8::new(self.flags.load(Ordering::Relaxed)), + // on clone we have to reset the content lock + #[cfg(not(target_os = "windows"))] + content: OnceLock::new(), + } + } +} + +/// Single-block read used by the binary classifier. Most binaries reveal a +/// NUL byte within the first filesystem block, so 16 KB lets one read settle +/// the classification for typical files while keeping the scratch buffer +/// small enough to live on the stack. +pub const BINARY_CLASSIFICATION_CHUNK_SIZE: usize = 16 * 1024; + +/// A file is treated as binary if any NUL byte appears in the scanned prefix. +#[inline] +pub(crate) fn detect_binary_content(content: &[u8]) -> bool { + memchr::memchr(0, content).is_some() +} + +impl FileItem { + pub fn new_raw( + filename_start: u16, + size: u64, + modified: u64, + git_status: Option, + is_binary: bool, + ) -> Self { + let mut flags = 0u8; + if is_binary { + flags |= FileItemFlags::BINARY; + } + + let mut path = crate::simd_path::ChunkedString::empty(); + path.filename_offset = filename_start; + + Self { + path, + parent_dir_index: u32::MAX, + size, + modified, + access_frecency_score: 0, + modification_frecency_score: 0, + git_status, + flags: AtomicU8::new(flags), + #[cfg(not(target_os = "windows"))] + content: OnceLock::new(), + } + } + + /// Returns an absolute path of the file + pub fn absolute_path(&self, arena: impl FFFStringStorage, base_path: &Path) -> PathBuf { + let mut buf = [0u8; PATH_BUF_SIZE]; + let rel = self.path.read_to_buf(arena.arena_for(self), &mut buf); + base_path.join(rel) + } + + pub(crate) fn set_path(&mut self, path: crate::simd_path::ChunkedString) { + self.path = path; + } + + pub fn dir_str(&self, arena: impl FFFStringStorage) -> String { + let mut s = String::with_capacity(64); + self.path.write_dir_to(arena.arena_for(self), &mut s); + s + } + + pub(crate) fn write_dir_str(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_dir_to(arena, out); + } + + pub fn file_name(&self, arena: impl FFFStringStorage) -> String { + let mut s = String::with_capacity(32); + self.path.write_filename_to(arena.arena_for(self), &mut s); + s + } + + pub(crate) fn write_file_name_from_arena(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_filename_to(arena, out); + } + + pub fn relative_path(&self, arena: impl FFFStringStorage) -> String { + let mut s = String::with_capacity(64); + self.path.write_to_string(arena.arena_for(self), &mut s); + s + } + + pub(crate) fn write_relative_path_from_arena(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_to_string(arena, out); + } + + pub fn relative_path_len(&self) -> usize { + self.path.byte_len as usize + } + + pub fn filename_offset_in_relative_path(&self) -> usize { + self.path.filename_offset as usize + } + + pub(crate) fn relative_path_eq(&self, arena: ArenaPtr, other: &str) -> bool { + if other.len() != self.path.byte_len as usize { + return false; + } + let mut buf = [0u8; 512]; + let mine = self.path.read_to_buf(arena, &mut buf); + mine == other + } + + pub(crate) fn relative_path_starts_with(&self, arena: ArenaPtr, prefix: &str) -> bool { + let mut buf = [0u8; PATH_BUF_SIZE]; + let path = self.path.read_to_buf(arena, &mut buf); + path.starts_with(prefix) + } + + /// Write `base_path + '/' + relative_path` into `buf` and return it + /// as `&Path`. Takes a fixed-size array so the buffer can live on + /// the stack (no heap allocation, no bounds checks in the hot loop). + pub(crate) fn write_absolute_path<'a>( + &self, + arena: ArenaPtr, + base_path: &Path, + buf: &'a mut [u8; PATH_BUF_SIZE], + ) -> &'a Path { + let base = base_path.as_os_str().as_encoded_bytes(); + let base_len = base.len(); + buf[..base_len].copy_from_slice(base); + let sep_len = if base_len > 0 && base[base_len - 1] != std::path::MAIN_SEPARATOR as u8 { + buf[base_len] = std::path::MAIN_SEPARATOR as u8; + 1 + } else { + 0 + }; + + let base_end_idx = base_len + sep_len; + let relative_portion_str = self.path.read_to_buf(arena, &mut buf[base_end_idx..]); + let rel_len = relative_portion_str.len(); + let total = base_end_idx + rel_len; + // Stored relative paths are '/'-canonical; rewrite to the OS-native + // separator so the result matches git-cache keys, the frecency DB, and + // Win32 file APIs. No-op off Windows. + crate::path_utils::nativize_slashes_in_place(&mut buf[base_end_idx..total]); + Path::new(unsafe { std::str::from_utf8_unchecked(&buf[..total]) }) + } + + /// Write the relative path into `buf` and NUL-terminate, returning + /// a `&CStr`. Fixed-size array so the buffer is stack-allocatable. + /// + /// Paired with a parent-directory fd this eliminates the per-file + /// absolute-path memcpy: `openat(dir_fd, cstr.as_ptr(), O_RDONLY)` + /// resolves the name relative to `dir_fd`. Unix-only. + #[cfg(unix)] + pub(crate) fn write_relative_cstr<'a>( + &self, + arena: ArenaPtr, + buf: &'a mut [u8; PATH_BUF_SIZE], + ) -> &'a std::ffi::CStr { + // Reserve the last byte for the NUL terminator. + let rel = self.path.read_to_buf(arena, &mut buf[..PATH_BUF_SIZE - 1]); + let n = rel.len(); + buf[n] = 0; + // SAFETY: `buf[..=n]` ends with the NUL we just wrote and + // filesystem paths never contain interior NULs. + unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(&buf[..=n]) } + } + + #[inline] + pub fn total_frecency_score(&self) -> i32 { + self.access_frecency_score as i32 + self.modification_frecency_score as i32 + } + + #[allow(dead_code)] + #[inline] + pub(crate) fn is_likely_hot(&self) -> bool { + self.access_frecency_score > 0 || self.git_status.is_some() + } + + /// Reads a fixed bytes count from the file optimized for quick speed of opening + #[inline] + pub(crate) fn read_trimmed_into_buf( + &self, + base_fd: i32, + base_path: &Path, + arena: ArenaPtr, + path_buf: &mut [u8; PATH_BUF_SIZE], + buf: &mut [u8], + ) -> usize { + #[cfg(unix)] + { + self.read_into_buf_unix(base_fd, base_path, arena, path_buf, buf) + } + #[cfg(not(unix))] + { + let _ = base_fd; + self.read_into_buf_std(base_path, arena, path_buf, buf) + } + } + + #[cfg(unix)] + fn read_into_buf_unix( + &self, + base_fd: libc::c_int, + base_path: &Path, + arena: ArenaPtr, + path_buf: &mut [u8; PATH_BUF_SIZE], + buf: &mut [u8], + ) -> usize { + let fd = if base_fd >= 0 { + let relative_path = self.write_relative_cstr(arena, path_buf); + // SAFETY: `relative_path` is NUL-terminated, `base_fd` is a + // valid directory descriptor owned by the caller. + unsafe { libc::openat(base_fd, relative_path.as_ptr(), libc::O_RDONLY) } + } else { + use std::os::unix::io::IntoRawFd; + let abs = self.write_absolute_path(arena, base_path, path_buf); + match std::fs::File::open(abs) { + Ok(f) => f.into_raw_fd(), + Err(e) => { + tracing::error!(?e, "Failed to fopen file"); + return 0; + } + } + }; + if fd < 0 { + return 0; + } + + let mut filled = 0usize; + while filled < buf.len() { + // SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a + // valid writable slice for `buf.len() - filled` bytes. + let n = unsafe { + libc::read( + fd, + buf[filled..].as_mut_ptr() as *mut libc::c_void, + (buf.len() - filled) as libc::size_t, + ) + }; + if n <= 0 { + break; + } + filled += n as usize; + } + + // SAFETY: matching close for the owned descriptor. + unsafe { libc::close(fd) }; + filled + } + + #[cfg(not(unix))] + fn read_into_buf_std( + &self, + base_path: &Path, + arena: ArenaPtr, + path_buf: &mut [u8; PATH_BUF_SIZE], + buf: &mut [u8], + ) -> usize { + let abs = self.write_absolute_path(arena, base_path, path_buf); + let Ok(mut f) = std::fs::File::open(abs) else { + return 0; + }; + let mut filled = 0usize; + while filled < buf.len() { + match f.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(_) => return 0, + } + } + filled + } + + #[inline] + pub fn is_binary(&self) -> bool { + self.flags.load(Ordering::Relaxed) & FileItemFlags::BINARY != 0 + } + + #[inline] + pub fn set_binary(&self, val: bool) { + if val { + self.flags + .fetch_or(FileItemFlags::BINARY, Ordering::Relaxed); + } else { + self.flags + .fetch_and(!FileItemFlags::BINARY, Ordering::Relaxed); + } + } + + /// Chunked classifier of the binary content of the file chunk by chunk + /// accepts path which to reuse the allocated buffer for absolute path read + pub(crate) fn detect_binary_per_byte(&self, path: &Path, chunk: &mut [u8]) { + if self.size == 0 { + return; + } + + let Ok(mut file) = std::fs::OpenOptions::new() + .write(false) + .read(true) + .open(path) + else { + tracing::error!(path = ?path.display(), "Failed to open indexed file"); + return; + }; + + loop { + match file.read(chunk) { + Ok(0) => break, + Err(e) => { + tracing::error!(?e, "Failed to read file chunk"); + break; + } + Ok(n) => { + if detect_binary_content(&chunk[..n]) { + self.set_binary(true); + } + } + } + } + } + + #[inline] + pub fn is_deleted(&self) -> bool { + self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0 + } + + #[inline] + #[doc(hidden)] + /// Don't use it, use FilePicker::delete_file + pub fn set_deleted(&self, val: bool) { + if val { + self.flags + .fetch_or(FileItemFlags::DELETED, Ordering::Relaxed); + } else { + self.flags + .fetch_and(!FileItemFlags::DELETED, Ordering::Relaxed); + } + } + + #[inline] + pub fn is_overflow(&self) -> bool { + self.flags.load(Ordering::Relaxed) & FileItemFlags::OVERFLOW != 0 + } + + #[inline] + pub fn set_overflow(&self, val: bool) { + if val { + self.flags + .fetch_or(FileItemFlags::OVERFLOW, Ordering::Relaxed); + } else { + self.flags + .fetch_and(!FileItemFlags::OVERFLOW, Ordering::Relaxed); + } + } +} + +impl FileItem { + /// Invalidate the cached mmap content, has to be called every time the file is updated. + /// + /// Call this when the background watcher detects that the file has been modified. + /// On Unix, a file that is truncated while mapped can cause SIGBUS. On Windows, + /// the stale buffer simply won't reflect the new contents. In both cases, + /// invalidating ensures a fresh read on the next access. + #[cfg(not(target_os = "windows"))] + pub fn invalidate_mmap(&mut self, budget: &ContentCacheBudget) { + if self.content.get().is_some() { + budget.cached_count.fetch_sub(1, Ordering::Relaxed); + budget.cached_bytes.fetch_sub(self.size, Ordering::Relaxed); + } + + self.content = OnceLock::new(); + } + + #[cfg(target_os = "windows")] + pub fn invalidate_mmap(&mut self, _: &ContentCacheBudget) {} + + pub fn update_metadata( + &mut self, + budget: &ContentCacheBudget, + modified_secs: Option, + new_size: Option, + ) { + if let Some(modified) = modified_secs + && self.modified < modified + { + self.modified = modified; + } + + self.invalidate_mmap(budget); + + if let Some(size) = new_size { + self.size = size; + } + } + + /// Get the cached file contents or lazily load and cache them. + /// + /// Returns `None` if the file is too large, empty, can't be opened, **or + /// the cache budget is exhausted**. Callers that need content regardless + /// of the budget should use [`get_content_for_search`]. + /// + /// After the first call, this is lock-free (just an atomic load + pointer deref). + /// + /// On Windows we never back this cache — `memmap2` would require a full + /// `std::fs::read` heap copy and the OS page cache already absorbs repeat + /// reopens. Returning `None` keeps callers on the scratch-read slow path + /// and avoids duplicating every indexed file on the heap. + #[cfg(target_os = "windows")] + pub(crate) fn get_cached_content( + &self, + _arena: ArenaPtr, + _base_path: &Path, + _budget: &ContentCacheBudget, + ) -> Option<&[u8]> { + None + } + + /// Returns a reference to a cached mmap of the file's contents. + /// + /// SAFETY-CRITICAL: callers must hold the picker read lock for as long as the returned slice is in use. + #[cfg(not(target_os = "windows"))] + pub(crate) fn get_cached_content( + &self, + arena: ArenaPtr, + base_path: &Path, + budget: &ContentCacheBudget, + ) -> Option<&[u8]> { + if let Some(content) = self.content.get() { + return Some(content); + } + + if self.size < MMAP_THRESHOLD || self.size > budget.max_file_size { + return None; + } + + // Check cache budget before creating a new persistent cache entry. + let count = budget.cached_count.load(Ordering::Relaxed); + let bytes = budget.cached_bytes.load(Ordering::Relaxed); + let max_files = budget.max_files; + let max_bytes = budget.max_bytes; + if count >= max_files || bytes + self.size > max_bytes { + return None; + } + + let path = self.absolute_path(arena, base_path); + let file = std::fs::File::open(&path).ok()?; + // SAFETY: the mmap is backed by the kernel page cache and reflects + // file updates; the only risk is SIGBUS on a concurrent truncate, + // which the watcher mitigates by invalidating on modification. + let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?; + let result = self.content.get_or_init(|| mmap); + + budget.cached_count.fetch_add(1, Ordering::Relaxed); + budget.cached_bytes.fetch_add(self.size, Ordering::Relaxed); + + Some(result) + } + + /// Get file content for searching — **always returns content** for eligible + /// files, even when the persistent cache budget is exhausted. + /// + /// The caller provides a reusable `path_buf` (pre-filled with `base_path/`) + /// and its `base_len` to avoid allocations when constructing the absolute path. + #[inline] + pub(crate) fn get_content_for_search<'a>( + &'a self, + buf: &'a mut Vec, + #[cfg_attr(target_os = "windows", allow(unused_variables))] mmap_slot: &'a mut MmapSlot, + arena: ArenaPtr, + base_path: &Path, + budget: &ContentCacheBudget, + ) -> Option<&'a [u8]> { + #[cfg(not(target_os = "windows"))] + { + // Fast path: persistent cache hit (zero-copy). Safe here because + // grep callers hold the picker read lock for the lifetime of the + // returned slice — see [`Self::get_cached_content`] safety note. + if let Some(cached) = self.get_cached_content(arena, base_path, budget) { + return Some(cached); + } + } + + let max_file_size = budget.max_file_size; + if self.is_binary() || self.size == 0 || self.size > max_file_size { + return None; + } + + let abs = self.absolute_path(arena, base_path); + + #[cfg(not(target_os = "windows"))] + if self.size >= FRESH_MMAP_THRESHOLD { + let file = std::fs::File::open(&abs).ok()?; + let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?; + let stored = mmap_slot.insert(mmap); + return Some(&stored[..]); + } else { + let _ = (mmap_slot, arena); + } + + let len = self.size as usize; + buf.resize(len, 0); + + let mut file = std::fs::File::open(&abs).ok()?; + file.read_exact(buf).ok()?; + Some(buf.as_slice()) + } +} + +/// Per-thread scratch slot owning a transient mmap returned from +/// [`FileItem::get_content_for_search`]. `Option` on Unix, +/// unit on Windows where mmap is unused. +#[cfg(not(target_os = "windows"))] +pub type MmapSlot = Option; +#[cfg(target_os = "windows")] +pub type MmapSlot = (); + +impl Constrainable for FileItem { + #[inline] + fn write_file_name(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_filename_to(arena, out); + } + + #[inline] + fn write_relative_path(&self, arena: ArenaPtr, out: &mut String) { + self.path.write_to_string(arena, out); + } + + #[inline] + fn git_status(&self) -> Option { + self.git_status + } + + #[inline] + fn is_overflow(&self) -> bool { + FileItem::is_overflow(self) + } +} + +#[derive(Debug, Clone, Default)] +pub struct Score { + pub total: i32, + pub base_score: i32, + pub filename_bonus: i32, + pub special_filename_bonus: i32, + pub frecency_boost: i32, + pub git_status_boost: i32, + pub distance_penalty: i32, + pub current_file_penalty: i32, + pub combo_match_boost: i32, + pub path_alignment_bonus: i32, + pub exact_match: bool, + pub match_type: &'static str, +} + +#[derive(Debug, Clone, Copy)] +pub struct PaginationArgs { + pub offset: usize, + pub limit: usize, +} + +impl Default for PaginationArgs { + fn default() -> Self { + Self { + offset: 0, + limit: 100, + } + } +} + +#[derive(Debug, Clone)] +pub struct ScoringContext<'a> { + pub query: &'a FFFQuery<'a>, + pub project_path: Option<&'a Path>, + pub current_file: Option<&'a str>, + pub max_typos: u16, + pub max_threads: usize, + pub last_same_query_match: Option, + pub combo_boost_score_multiplier: i32, + pub min_combo_count: u32, + pub pagination: PaginationArgs, +} + +impl ScoringContext<'_> { + pub fn effective_query(&self) -> &str { + match &self.query.fuzzy_query { + FuzzyQuery::Text(t) => t, + FuzzyQuery::Parts(parts) if !parts.is_empty() => parts[0], + _ => self.query.raw_query.trim(), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct SearchResult<'a> { + pub items: Vec<&'a FileItem>, + pub scores: Vec, + pub total_matched: usize, + pub total_files: usize, + pub location: Option, +} + +/// Search result for directory-only fuzzy search. +#[derive(Debug, Clone, Default)] +pub struct DirSearchResult<'a> { + pub items: Vec<&'a DirItem>, + pub scores: Vec, + pub total_matched: usize, + pub total_dirs: usize, +} + +/// A single item in a mixed (files + directories) search result. +#[derive(Debug, Clone)] +pub enum MixedItemRef<'a> { + File(&'a FileItem), + Dir(&'a DirItem), +} + +/// Search result for mixed (files + directories) fuzzy search. +/// Items are interleaved by total score in descending order. +#[derive(Debug, Clone, Default)] +pub struct MixedSearchResult<'a> { + pub items: Vec>, + pub scores: Vec, + pub total_matched: usize, + pub total_files: usize, + pub total_dirs: usize, + pub location: Option, +} + +impl Default for MixedItemRef<'_> { + fn default() -> Self { + // Should never be used, exists only for Default derive on MixedSearchResult + unreachable!("MixedItemRef::default should not be called") + } +} + +#[derive(Debug)] +pub struct ContentCacheBudget { + pub max_files: usize, + pub max_bytes: u64, + pub max_file_size: u64, + pub cached_count: AtomicUsize, + pub cached_bytes: AtomicU64, +} + +impl ContentCacheBudget { + pub fn unlimited() -> Self { + Self { + max_files: usize::MAX, + max_bytes: u64::MAX, + max_file_size: MAX_FFFILE_SIZE, + cached_count: AtomicUsize::new(0), + cached_bytes: AtomicU64::new(0), + } + } + + pub fn zero() -> Self { + Self { + max_files: 0, + max_bytes: 0, + max_file_size: 0, + cached_count: AtomicUsize::new(0), + cached_bytes: AtomicU64::new(0), + } + } + + // Byte budget + pub fn is_exhausted(&self) -> bool { + self.cached_count.load(Ordering::Relaxed) >= self.max_files + || self.cached_bytes.load(Ordering::Relaxed) >= self.max_bytes + } + + pub fn new_for_repo(file_count: usize) -> Self { + let max_files = if file_count > 50_000 { + 5_000 + } else if file_count > 10_000 { + 10_000 + } else { + 30_000 // effectively unlimited for small repos + }; + + let max_bytes = if file_count > 50_000 { + 128 * 1024 * 1024 // 128 MB + } else if file_count > 10_000 { + 256 * 1024 * 1024 // 256 MB + } else { + MAX_CACHED_CONTENT_BYTES // 512 MB + }; + + Self { + max_files, + max_bytes, + max_file_size: MAX_FFFILE_SIZE, + cached_count: AtomicUsize::new(0), + cached_bytes: AtomicU64::new(0), + } + } + + /// Build a budget from caller-supplied overrides. + /// + /// Each argument is a cap; `0` means "use the library default for that + /// cap" (inherits from [`Self::default`], which is `new_for_repo(30_000)`). + /// Returns `None` when every cap is `0`, signalling to the picker that it + /// should auto-size the budget from the final scanned file count rather + /// than applying an explicit override. + pub fn from_overrides(max_files: usize, max_bytes: u64, max_file_size: u64) -> Option { + if max_files == 0 && max_bytes == 0 && max_file_size == 0 { + return None; + } + + let mut budget = Self::default(); + if max_files > 0 { + budget.max_files = max_files; + } + if max_bytes > 0 { + budget.max_bytes = max_bytes; + } + if max_file_size > 0 { + budget.max_file_size = max_file_size; + } + Some(budget) + } + + pub fn reset(&self) { + self.cached_count.store(0, Ordering::Relaxed); + self.cached_bytes.store(0, Ordering::Relaxed); + } +} + +impl Default for ContentCacheBudget { + fn default() -> Self { + Self::new_for_repo(30_000) + } +} diff --git a/crates/fff-core/src/walk/mod.rs b/crates/fff-core/src/walk/mod.rs new file mode 100644 index 0000000..a533f0b --- /dev/null +++ b/crates/fff-core/src/walk/mod.rs @@ -0,0 +1,146 @@ +//! Filesystem traversal backend. Selects one implementation at compile time: +//! - `zlob`: zlob's native parallel walker (requires the Zig toolchain). +//! - `ripgrep`: the `ignore` crate (ripgrep's walker), used by default. +//! +//! Both expose [`walk_collect_files`] with identical semantics so the rest of +//! the crate stays backend-agnostic. + +use crate::types::FileItem; +use std::path::Path; + +#[cfg(feature = "zlob")] +mod zlob; +#[cfg(feature = "zlob")] +pub(crate) use zlob::walk_collect_files; + +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +mod ripgrep; +#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))] +pub(crate) use ripgrep::walk_collect_files; + +pub(crate) struct WalkOutput { + pub(crate) pairs: Vec<(FileItem, String)>, + pub(crate) ignore_rules: Option, +} + +pub(crate) struct WalkIgnoreRules { + #[cfg(feature = "zlob")] + inner: ::zlob::walk::WalkerOutcomeRules, + #[cfg(not(feature = "zlob"))] + _never: std::convert::Infallible, +} + +// SAFETY: the underlying storage is immutable, heap-owned, and thread-safe to +// read from concurrently (mirrors zlob's `IgnoreRules: Send + Sync`). +unsafe impl Send for WalkIgnoreRules {} +unsafe impl Sync for WalkIgnoreRules {} + +impl std::fmt::Debug for WalkIgnoreRules { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("WalkIgnoreRules") + } +} + +// In ripgrep builds `WalkIgnoreRules` is never constructed (the `_never` +// field is uninhabited), so its methods are legitimately dead there. +#[cfg_attr(not(feature = "zlob"), allow(dead_code))] +impl WalkIgnoreRules { + /// Returns `true` if the provided path is ignored by the collected rule set + /// + /// `relative_path` has to be relative to the walker's provided base path + pub(crate) fn is_ignored(&self, relative_path: &Path) -> bool { + #[cfg(feature = "zlob")] + { + self.inner + .rules() + .is_some_and(|rules| rules.is_ignored(relative_path)) + } + #[cfg(not(feature = "zlob"))] + { + let _ = relative_path; + match self._never {} + } + } + + // The old `is_ignored_untrusted` variant was folded away when zlob's + // ignore matcher moved to full ancestor enumeration — trailing-slash + // sniffing on the input is now sufficient for external queries. +} + +#[cfg(test)] +mod tests { + use super::walk_collect_files; + use std::fs; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + // Backend-agnostic parity check: both the zlob and ripgrep walkers must + // respect .gitignore, skip hidden files in a git repo, and surface the + // expected file set with a correct synced count. + #[test] + fn collects_files_respecting_gitignore() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + fs::create_dir(root.join(".git")).unwrap(); + fs::create_dir(root.join("src")).unwrap(); + fs::create_dir(root.join("target")).unwrap(); + fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap(); + fs::write(root.join("Cargo.toml"), "x").unwrap(); + fs::write(root.join("debug.log"), "").unwrap(); + fs::write(root.join("src/main.rs"), "fn main() {}").unwrap(); + fs::write(root.join("target/out.bin"), "bin").unwrap(); + + let counter = Arc::new(AtomicUsize::new(0)); + let out = walk_collect_files(root, true, false, 1, &counter).unwrap(); + + let mut names: Vec = out.pairs.into_iter().map(|(_, rel)| rel).collect(); + names.sort(); + + assert!(names.contains(&"Cargo.toml".to_string())); + assert!(names.iter().any(|n| n.ends_with("main.rs"))); + // target/ and *.log are gitignored; .git/ is skipped. + assert!(!names.iter().any(|n| n.contains("target"))); + assert!(!names.iter().any(|n| n.ends_with(".log"))); + assert!(!names.iter().any(|n| n.contains(".git/"))); + assert_eq!(counter.load(Ordering::Relaxed), names.len()); + } + + // Non-git roots prune known non-code directories (node_modules). + #[test] + fn prunes_non_code_dirs_for_non_git_root() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + fs::create_dir(root.join("node_modules")).unwrap(); + fs::write(root.join("node_modules/lib.js"), "x").unwrap(); + fs::write(root.join("index.js"), "x").unwrap(); + + let counter = Arc::new(AtomicUsize::new(0)); + let out = walk_collect_files(root, false, false, 1, &counter).unwrap(); + let names: Vec = out.pairs.into_iter().map(|(_, rel)| rel).collect(); + + assert!(names.iter().any(|n| n.ends_with("index.js"))); + assert!(!names.iter().any(|n| n.contains("node_modules"))); + } + + // Only the zlob backend surfaces reusable ignore rules; they must match + // the same tree the walk respected. + #[cfg(feature = "zlob")] + #[test] + fn surfaces_reusable_ignore_rules() { + use std::path::Path; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + fs::create_dir(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap(); + fs::write(root.join("Cargo.toml"), "x").unwrap(); + + let counter = Arc::new(AtomicUsize::new(0)); + let out = walk_collect_files(root, true, false, 1, &counter).unwrap(); + + let rules = out.ignore_rules.expect("zlob surfaces ignore rules"); + assert!(rules.is_ignored(Path::new("target/"))); + assert!(rules.is_ignored(Path::new("debug.log"))); + assert!(!rules.is_ignored(Path::new("Cargo.toml"))); + } +} diff --git a/crates/fff-core/src/walk/ripgrep.rs b/crates/fff-core/src/walk/ripgrep.rs new file mode 100644 index 0000000..249de2d --- /dev/null +++ b/crates/fff-core/src/walk/ripgrep.rs @@ -0,0 +1,72 @@ +use crate::background_watcher::is_git_file; +use crate::ignore::non_git_repo_overrides; +use crate::types::FileItem; +use crate::walk::WalkOutput; +use ignore::WalkBuilder; +use std::path::Path; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +#[tracing::instrument(skip_all, name = "ripgrep walker", level = "info")] +pub(crate) fn walk_collect_files( + base_path: &Path, + is_git_repo: bool, + follow_symlinks: bool, + threads: usize, + synced_files_count: &Arc, +) -> crate::Result { + let mut walk_builder = WalkBuilder::new(base_path); + walk_builder + // this is a very important guard for the user opening ~/ or other root non-git dir + .hidden(!is_git_repo) + .git_ignore(true) + .git_exclude(true) + .git_global(true) + .ignore(true) + .follow_links(follow_symlinks) + .threads(threads); + + if !is_git_repo && let Some(overrides) = non_git_repo_overrides(base_path) { + walk_builder.overrides(overrides); + } + + let walker = walk_builder.build_parallel(); + + let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new()); + walker.run(|| { + let pairs = &pairs; + let counter = Arc::clone(synced_files_count); + let base_path = base_path.to_path_buf(); + + Box::new(move |result| { + let Ok(entry) = result else { + return ignore::WalkState::Continue; + }; + + if entry.file_type().is_some_and(|ft| ft.is_file()) { + let path = entry.path(); + + // Ignore walkers sometimes surface files inside `.git/` + // when the base is itself a git repo — skip them. + if is_git_file(path) { + return ignore::WalkState::Continue; + } + + let metadata = entry.metadata().ok(); + let (file_item, rel_path) = + FileItem::new_from_walk(path, &base_path, None, metadata.as_ref()); + + pairs.lock().push((file_item, rel_path)); + counter.fetch_add(1, Ordering::Relaxed); + } + ignore::WalkState::Continue + }) + }); + + Ok(WalkOutput { + pairs: pairs.into_inner(), + ignore_rules: None, + }) +} diff --git a/crates/fff-core/src/walk/zlob.rs b/crates/fff-core/src/walk/zlob.rs new file mode 100644 index 0000000..0978ad1 --- /dev/null +++ b/crates/fff-core/src/walk/zlob.rs @@ -0,0 +1,111 @@ +//! Filesystem traversal backed by zlob's native parallel walker. +//! Active when the `zlob` feature is enabled (requires the Zig toolchain). + +use crate::file_picker::is_known_binary_extension_basename; +use crate::ignore::IGNORED_DIRS; +use crate::types::FileItem; +use crate::walk::{WalkIgnoreRules, WalkOutput}; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use zlob::walk::{WalkBuilder, WalkFlags, WalkMetadata, WalkState}; + +const PROGRESS_STEP: usize = 13; + +#[tracing::instrument(skip_all, name = "zlob walker", level = "info")] +pub(crate) fn walk_collect_files( + base_path: &Path, + is_git_repo: bool, + follow_symlinks: bool, + threads: usize, + synced_files_count: &Arc, +) -> crate::Result { + // gitignore on; skip hidden on non-git roots (so `~/` doesn't recurse into + // ~/.cache, ~/.config, etc.); optionally follow symlinks. + let mut flags = WalkFlags::GITIGNORE; + if !is_git_repo { + flags |= WalkFlags::SKIP_HIDDEN; + } + if follow_symlinks { + flags |= WalkFlags::FOLLOW_SYMLINKS; + } + + let mut builder = WalkBuilder::new(base_path) + .map_err(|e| crate::Error::WalkFailed(format!("WalkBuilder::new: {e:?}")))?; + builder + .options(flags) + .threads(threads) + // Bulk-fetch the only metadata FileItem needs; zlob never stats more. + .metadata(WalkMetadata::SIZE | WalkMetadata::MTIME); + + if !is_git_repo + && !IGNORED_DIRS.is_empty() + && let Err(e) = builder.extra_ignore(IGNORED_DIRS) + { + // Interior NUL in one of the extra_ignore patterns would fail + // here — treat as if no extras were supplied rather than + // aborting the whole walk. + tracing::warn!(?e, "zlob extra_ignore rejected; walking without it"); + } + + let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new()); + + let outcome = match builder.run(|entry| { + if !entry.is_file() { + return WalkState::Continue; + } + let rel_bytes = entry.relative_path_bytes(); + + // `basename()` returns `&str` for files only. + let basename = entry.basename().unwrap_or(""); + let is_binary = is_known_binary_extension_basename(basename); + + let size = entry.size().unwrap_or(0); + // zlob reports mtime in ns since the Unix epoch; FileItem wants secs. + let modified = entry + .modified_ns() + .map(|ns| (ns / 1_000_000_000).max(0) as u64) + .unwrap_or(0); + + let basename_offset = entry.basename_offset_in_relative(); + // zlob emits '/'-separated relative paths, which is fff's canonical + // internal form on every platform — store them verbatim. + let rel_str = String::from_utf8_lossy(rel_bytes).into_owned(); + let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary); + + let mut guard = pairs.lock(); + guard.push((item, rel_str)); + let n = guard.len(); + drop(guard); + + if n % PROGRESS_STEP == 0 { + synced_files_count.store(n, Ordering::Relaxed); + } + + WalkState::Continue + }) { + Ok(outcome) => outcome, + Err(e) => { + // Preserve whatever we collected before the failure so the caller + // can still surface a partial index instead of nothing. + tracing::error!(?e, "zlob walk failed"); + return Err(crate::Error::WalkFailed(format!("{e:?}"))); + } + }; + + let pairs = pairs.into_inner(); + // Always report the exact final total regardless of the last step. + synced_files_count.store(pairs.len(), Ordering::Relaxed); + + // Retain the ignore rules only when the walk actually gathered some + // (git roots with .gitignore/.ignore). Otherwise callers fall back. + let ignore_rules = outcome + .rules() + .is_some() + .then(|| WalkIgnoreRules { inner: outcome }); + + Ok(WalkOutput { + pairs, + ignore_rules, + }) +} diff --git a/crates/fff-core/tests/bigram_overlay_coherence_test.rs b/crates/fff-core/tests/bigram_overlay_coherence_test.rs new file mode 100644 index 0000000..6cee64e --- /dev/null +++ b/crates/fff-core/tests/bigram_overlay_coherence_test.rs @@ -0,0 +1,1656 @@ +//! Stress tests for the bigram overlay layer. +//! +//! These tests create a real git repository with many files, build the bigram +//! index, then perform loops of creates / edits / deletes and verify that grep +//! always returns correct results through the overlay. Finally, a git commit + +//! rescan cycle verifies the index is rebuilt cleanly. +//! +//! # Known bugs exposed by these tests +//! +//! 1. **Overflow files invisible to grep** — `grep_search` only iterates +//! base-file bits from the bigram candidate bitset. Overflow files +//! (index >= base_count) are never included. `BigramOverlay::query_added` +//! exists but is dead code — never called from the grep path. +//! See: grep.rs lines ~1787-1855. + +mod overflow_frecency_segfault; + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{ + FilePickerOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency, +}; + +/// Stress test: 200 base files, 3 rounds of edits + deletes. New files +/// are tracked but NOT verified via grep (see Group 3 for that bug). +#[test] +fn bigram_overlay_coherence_stress_base_edits_and_deletes() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let initial_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + // Sanity: all 200 tokens findable. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for (_, token) in &initial_files { + assert!( + grep_count(picker, token) >= 1, + "Initial token {token} should be findable" + ); + } + } + + let mut live_tokens: Vec<(String, String)> = initial_files.clone(); + let mut dead_tokens: Vec = Vec::new(); + + // Sleep so mtime advances past the scan snapshot. + std::thread::sleep(Duration::from_millis(1100)); + + for round in 0..3 { + // -- DELETE: remove first 5 live base files -- + let delete_count = 5.min(live_tokens.len()); + let to_delete: Vec<(String, String)> = live_tokens.drain(..delete_count).collect(); + for (name, token) in &to_delete { + let path = base.join(name); + fs::remove_file(&path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.remove_file_by_path(&path), + "round {round}: remove({name}) should succeed" + ); + } + dead_tokens.push(token.clone()); + } + + // -- EDIT: modify next 5 live base files -- + let edit_count = 5.min(live_tokens.len()); + for i in 0..edit_count { + let (ref name, ref mut old_token) = live_tokens[i]; + let new_token = format!("EDITED_R{round}_{i:04}"); + write_file_with_token(base, name, &new_token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.handle_create_or_modify(base.join(name)).is_some(), + "round {round}: modify({name}) should succeed" + ); + } + dead_tokens.push(old_token.clone()); + *old_token = new_token; + } + + // -- VERIFY: all live base tokens findable, all dead tokens gone -- + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for (name, token) in &live_tokens { + let count = grep_count(picker, token); + assert!( + count >= 1, + "round {round}: live token {token} ({name}) should be findable, got {count}" + ); + } + + for token in &dead_tokens { + assert_eq!( + grep_count(picker, token), + 0, + "round {round}: dead token {token} should NOT be findable" + ); + } + } + } + + stop_picker(&shared_picker); +} + +/// Simulate a long editing session: 10 rounds of edits to 20 base files. +/// Only the latest content should be searchable. +#[test] +fn bigram_overlay_coherence_long_session_incremental_edits() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + let file_count = 20; + let edits_per_file = 10; + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + let mut latest_tokens: Vec = repo_files[..file_count] + .iter() + .map(|(_, token)| token.clone()) + .collect(); + + for edit_round in 0..edits_per_file { + for file_idx in 0..file_count { + let (ref name, _) = repo_files[file_idx]; + let new_token = format!("LONG_EDIT_F{file_idx:04}_R{edit_round:04}"); + write_file_with_token(base, name, &new_token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + } + latest_tokens[file_idx] = new_token; + } + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for (file_idx, token) in latest_tokens.iter().enumerate() { + let (ref name, _) = repo_files[file_idx]; + assert!( + grep_count(picker, token) >= 1, + "{name}: latest token {token} should be findable" + ); + } + + // Spot-check old tokens are gone. + for file_idx in [0, 5, 10, 15, 19] { + if file_idx >= file_count { + continue; + } + let (ref name, ref seed_token) = repo_files[file_idx]; + assert_eq!( + grep_count(picker, seed_token), + 0, + "{name}: seed token should not be findable" + ); + let mid = format!("LONG_EDIT_F{file_idx:04}_R0003"); + assert_eq!( + grep_count(picker, &mid), + 0, + "{name}: intermediate token should not be findable" + ); + } + } + + stop_picker(&shared_picker); +} + +/// Resurrect tombstoned base files: delete then re-create with new content. +#[test] +fn bigram_overlay_coherence_resurrect_tombstoned_file() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + let (ref target_name, ref target_seed_token) = repo_files[3]; + let target_path = base.join(target_name); + + // Delete. + fs::remove_file(&target_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&target_path); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!(grep_count(picker, target_seed_token), 0); + } + + // Re-create with new content. + write_file_with_token(base, target_name, "RESURRECTED_TOKEN"); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!(picker.handle_create_or_modify(&target_path).is_some()); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + grep_count(picker, "RESURRECTED_TOKEN") >= 1, + "resurrected token should be findable" + ); + assert_eq!( + grep_count(picker, target_seed_token), + 0, + "old token should not be findable after resurrect" + ); + } + + stop_picker(&shared_picker); +} + +/// Prove the overlay is doing real work for base file modifications. +/// Uses files with diverse content so the bigram index is discriminating. +#[test] +fn bigram_overlay_coherence_proves_contribution_for_modified_base() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Pick the 2nd file from the realistic repo to modify. + let (ref target_name, _) = repo_files[1]; + let target_path = base.join(target_name); + + // Replace with content containing a unique token that shares + // NO bigrams with the original thematic content. + let unique = "XYZZY_PLUGH_WIZARDRY"; + fs::write( + &target_path, + format!("fn beta() {{ println!(\"{unique}\"); }}\n"), + ) + .unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(&target_path); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let with_overlay = grep_count(picker, unique); + assert_eq!(with_overlay, 1, "overlay should find the new token"); + } + + stop_picker(&shared_picker); +} + +/// Rapidly create-then-delete the same base file path 10 times. +#[test] +fn bigram_overlay_coherence_rapid_create_delete_same_base_path() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + + // volatile.rs is added ON TOP of the realistic repo as a base file. + fs::write(base.join("volatile.rs"), "initial volatile content\n").unwrap(); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + // Sanity: a token from the realistic repo is findable. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let (_, ref token) = repo_files[0]; + assert!( + grep_count(picker, token) >= 1, + "sanity: realistic repo token should be findable" + ); + } + + std::thread::sleep(Duration::from_millis(1100)); + + let volatile_path = base.join("volatile.rs"); + + for cycle in 0..10 { + let token = format!("VOLATILE_CYCLE_{cycle:03}"); + + // Overwrite. + write_file_with_token(base, "volatile.rs", &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(&volatile_path); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + grep_count(picker, &token) >= 1, + "cycle {cycle}: {token} should be findable" + ); + if cycle > 0 { + let prev = format!("VOLATILE_CYCLE_{:03}", cycle - 1); + assert_eq!( + grep_count(picker, &prev), + 0, + "cycle {cycle}: previous token should be gone" + ); + } + } + + // Delete (tombstone). + fs::remove_file(&volatile_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&volatile_path); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!( + grep_count(picker, &token), + 0, + "cycle {cycle}: {token} should be gone after delete" + ); + } + + // Re-create for next cycle. + write_file_with_token(base, "volatile.rs", &format!("placeholder_{cycle}")); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(&volatile_path); + } + } + + stop_picker(&shared_picker); +} + +/// New overflow files should be searchable via grep. +/// +/// To trigger the bug, we need an effective bigram index (diverse file +/// content so prefiltering is actually active). With identical-template +/// files, ALL bigrams are either ubiquitous (in every file) or unique +/// (in only one file), so the index retains no columns and prefiltering +/// is always bypassed — masking the bug. +/// +/// The realistic repo seeds 200 files with 4 thematic groups so bigram +/// columns have moderate cardinality (retained by the index), making +/// the prefilter discriminating. +#[test] +fn bigram_overlay_coherence_overflow_files_searchable_via_grep() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + // Verify bigram index was built and is effective. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!(picker.bigram_index().is_some()); + assert!(picker.bigram_overlay().is_some()); + + // Sanity: a query with content from one file finds exactly that file. + assert!( + grep_count(picker, "REPO_TOKEN_0000") >= 1, + "sanity: known content should be findable" + ); + } + + std::thread::sleep(Duration::from_millis(1100)); + + // Add a new overflow file whose content has bigrams shared with + // existing files (e.g., "SELECT" shares bigrams with group A files). + let new_path = base.join("overflow_new.rs"); + fs::write( + &new_path, + "pub fn recalculate_velocity_delta() {\n \ + let v = SELECT_transaction();\n \ + println!(\"delta: {}\", v);\n}\n", + ) + .unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!(picker.handle_create_or_modify(&new_path).is_some()); + } + + // Overflow file is tracked. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!(picker.get_overflow_files().len(), 1); + } + + // BUG: grep_search's bigram candidate bitset only covers base files + // (indices 0..base_count). Overflow files at index >= base_count are + // never set as candidates. BigramOverlay::query_added() exists but + // is dead code — never called from the grep path. + // + // Search for content unique to the overflow file. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + grep_count(picker, "recalculate_velocity_delta") >= 1, + "BUG: overflow file content should be findable via grep but \ + bigram prefiltering skips overflow files" + ); + } + + stop_picker(&shared_picker); +} + +/// Mixed tombstones and overflow: delete base files AND add new ones. +/// Tombstone assertions pass; overflow grep assertions expose the bug. +#[test] +fn bigram_overlay_coherence_mixed_tombstones_and_overflow() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Delete first 15 base files. + let deleted_tokens: Vec = repo_files[..15] + .iter() + .map(|(_, token)| token.clone()) + .collect(); + for (name, _) in &repo_files[..15] { + let path = base.join(name); + fs::remove_file(&path).unwrap(); + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&path); + } + + // Add 15 new overflow files. + let mut new_tokens = Vec::new(); + for i in 0..15 { + let name = format!("replacement_{i:04}.rs"); + let token = format!("REPLACEMENT_{i:04}"); + write_file_with_token(base, &name, &token); + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(&name)); + new_tokens.push(token); + } + + { + // assert + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + // deleted tokens are gon + for token in &deleted_tokens { + assert_eq!( + grep_count(picker, token), + 0, + "deleted token {token} should not be findable" + ); + } + + // Surviving base files still findable. + for (name, token) in &repo_files[15..] { + assert!( + grep_count(picker, token) == 1, + "surviving {name} token should be findable" + ); + } + + // New tokens needs to be findable + for token in &new_tokens { + assert!( + grep_count(picker, token) == 1, + "BUG: overflow token {token} should be findable" + ); + } + } + + stop_picker(&shared_picker); +} + +#[test] +fn bigram_overlay_coherence_full_stress_loop_with_overflow() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let initial_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + let mut live_base: Vec<(String, String)> = initial_files; + let mut live_overflow: Vec<(String, String)> = Vec::new(); + let mut dead_tokens: Vec = Vec::new(); + + std::thread::sleep(Duration::from_millis(1100)); + + for round in 0..3 { + // -- DELETE 5 base files -- + let to_delete: Vec<_> = live_base.drain(..5).collect(); + for (name, token) in &to_delete { + let path = base.join(name); + fs::remove_file(&path).unwrap(); + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!(picker.remove_file_by_path(&path)); + dead_tokens.push(token.clone()); + } + + // -- EDIT 5 base files -- + for i in 0..5.min(live_base.len()) { + let (ref name, ref mut old_token) = live_base[i]; + let new_token = format!("EDITED_R{round}_{i:04}"); + write_file_with_token(base, name, &new_token); + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + dead_tokens.push(old_token.clone()); + *old_token = new_token; + } + + // -- CREATE 5 overflow files -- + for i in 0..5 { + let name = format!("new_r{round}_{i:04}.rs"); + let token = format!("NEWFILE_R{round}_{i:04}"); + write_file_with_token(base, &name, &token); + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(&name)); + live_overflow.push((name, token)); + } + + // -- VERIFY base files -- + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for (name, token) in &live_base { + assert!( + grep_count(picker, token) >= 1, + "round {round}: base token {token} ({name}) should be findable" + ); + } + + for token in &dead_tokens { + assert_eq!( + grep_count(picker, token), + 0, + "round {round}: dead token {token} should NOT be findable" + ); + } + + // BUG: overflow files not findable via grep. + for (name, token) in &live_overflow { + assert!( + grep_count(picker, token) >= 1, + "round {round}: BUG overflow token {token} ({name}) not findable via grep" + ); + } + } + } + + stop_picker(&shared_picker); +} + +/// Overflow files can be edited and deleted through the picker. +/// Verifies the overlay tracks overflow changes even though grep may not +/// search them (that's the separate bug in Group 2). +#[test] +fn bigram_overlay_coherence_overflow_file_edit_and_delete() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Add overflow files and track their paths. + let mut overflow_files: Vec<(PathBuf, String)> = Vec::new(); + for i in 0..10 { + let name = format!("overflow_{i}.rs"); + let token = format!("OVERFLOW_ORIG_{i}"); + let path = base.join(&name); + write_file_with_token(base, &name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(&path); + } + overflow_files.push((path, token)); + } + + // Verify overflow files are tracked in the picker. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!( + picker.get_overflow_files().len(), + 10, + "should have 10 overflow files" + ); + } + + // Edit all overflow files. + let mut edited_tokens = Vec::new(); + for (i, (path, _)) in overflow_files.iter().enumerate() { + let name = path.file_name().unwrap().to_str().unwrap(); + let new_token = format!("OVERFLOW_EDITED_{i}"); + write_file_with_token(base, name, &new_token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(path); + } + edited_tokens.push(new_token); + } + + // Delete first 5 overflow files. + for i in 0..5 { + let (ref path, _) = overflow_files[i]; + fs::remove_file(path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(path); + } + } + + // Verify 5 overflow remain live (tombstones still occupy slots by design — + // StableVec never shifts, so get_overflow_files().len() stays at 10). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let live = picker + .get_overflow_files() + .iter() + .filter(|f| !f.is_deleted()) + .count(); + assert_eq!( + live, 5, + "should have 5 live overflow files after deleting 5" + ); + } + + stop_picker(&shared_picker); +} + +// =================================================================== +// Group 4: Rescan after git commit. Tests that `trigger_rescan` picks +// up committed changes and that the file list is refreshed. +// =================================================================== + +/// After editing base files, committing, and rescanning, the new content +/// should be searchable. +/// +/// NOTE: `trigger_rescan` replaces `sync_data` which drops the bigram +/// index (it lives inside `FileSync`). After rescan, grep falls back to +/// full search (no bigram prefiltering) which is correct but slower. +#[test] +fn bigram_overlay_coherence_rescan_after_git_commit() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + let base_count = repo_files.len(); // 200 + git_init_and_commit(base); + + let (shared_picker, shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Edit 10 base files and add 5 new files. + let mut edited_tokens = Vec::new(); + for i in 0..10 { + let (ref name, _) = repo_files[i]; + let token = format!("PRE_COMMIT_EDIT_{i:04}"); + write_file_with_token(base, name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + } + edited_tokens.push(token); + } + + let mut new_tokens = Vec::new(); + for i in 0..5 { + let name = format!("committed_new_{i:04}.rs"); + let token = format!("COMMITTED_NEW_{i:04}"); + write_file_with_token(base, &name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(&name)); + } + new_tokens.push(token); + } + + // Phase 2: Commit and rescan. + git_add_and_commit(base, "batch edit"); + shared_picker + .trigger_full_rescan_async(&shared_frecency) + .expect("rescan should succeed"); + + // After trigger_rescan, sync_data is replaced (and bigram_index dropped + // with it). Wait for the synchronous scan to finish. + assert!( + shared_picker.wait_for_scan(Duration::from_secs(15)), + "Timed out waiting for scan to complete" + ); + + // Verify the file list is refreshed: all base_count + 5 files should + // be present as base files (not overflow, since they're committed). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!( + picker.get_files().len(), + base_count + 5, + "post-rescan: should have {} files ({} original + 5 new)", + base_count + 5, + base_count + ); + // After rescan, new files are in the base (no overflow). + assert_eq!( + picker.get_overflow_files().len(), + 0, + "post-rescan: should have 0 overflow files" + ); + } + + // Post-rescan grep verification. + // After rescan + bigram rebuild, edited tokens should be in the base + // index and findable without the overlay. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for token in &edited_tokens { + let with = grep_count(picker, token); + assert!( + with >= 1, + "post-rescan: edited token {token} should be findable" + ); + } + + for token in &new_tokens { + let with = grep_count(picker, token); + assert!( + with >= 1, + "post-rescan: new token {token} should be findable" + ); + } + } + + stop_picker(&shared_picker); +} + +// =================================================================== +// Group 5: Full lifecycle -- seed, overlay edits, commit, rescan, more +// overlay edits. Tests the complete workflow. +// =================================================================== + +/// seed -> index -> overlay edits -> commit -> rescan -> more overlay edits +#[test] +fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // -- Phase 1: Pre-commit overlay edits (base files only) -- + let mut phase1_tokens = Vec::new(); + for i in 0..10 { + let (ref name, _) = repo_files[i]; + let token = format!("PHASE1_EDIT_{i:04}"); + write_file_with_token(base, name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + } + phase1_tokens.push(token); + } + + // Delete some files (pick indices near the end). + let mut phase1_dead = Vec::new(); + for i in 195..200 { + let (ref name, ref token) = repo_files[i]; + let path = base.join(name); + fs::remove_file(&path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&path); + } + phase1_dead.push(token.clone()); + } + + // Verify phase 1 (base file operations only). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for token in &phase1_tokens { + assert!( + grep_count(picker, token) >= 1, + "phase1: {token} should be findable" + ); + } + for token in &phase1_dead { + assert_eq!( + grep_count(picker, token), + 0, + "phase1: {token} should be gone" + ); + } + } + + // -- Phase 2: Commit and rescan -- + git_add_and_commit(base, "phase 1 changes"); + shared_picker + .trigger_full_rescan_async(&shared_frecency) + .expect("rescan should succeed"); + + assert!( + shared_picker.wait_for_scan(Duration::from_secs(15)), + "Timed out waiting for scan to complete" + ); + + // Phase1 tokens should still be findable (now in base index). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for token in &phase1_tokens { + assert!( + grep_count(picker, token) >= 1, + "post-rescan: {token} should still be findable" + ); + } + for token in &phase1_dead { + assert_eq!( + grep_count(picker, token), + 0, + "post-rescan: {token} should not be findable" + ); + } + } + + std::thread::sleep(Duration::from_millis(1100)); + + // -- Phase 3: More overlay edits after rescan -- + let mut phase3_tokens = Vec::new(); + for i in 0..5 { + let (ref name, _) = repo_files[i]; + let token = format!("PHASE3_EDIT_{i:04}"); + write_file_with_token(base, name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + } + phase3_tokens.push(token); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for token in &phase3_tokens { + assert!( + grep_count(picker, token) >= 1, + "phase3: {token} should be findable" + ); + } + + // Phase1 tokens for files 0-4 were overwritten by phase3. + for i in 0..5 { + let old = format!("PHASE1_EDIT_{i:04}"); + assert_eq!( + grep_count(picker, &old), + 0, + "phase3: overwritten {old} should not be findable" + ); + } + + // Phase1 tokens for files 5-9 should still be there (in base). + for i in 5..10 { + let token = format!("PHASE1_EDIT_{i:04}"); + assert!( + grep_count(picker, &token) >= 1, + "phase3: {token} should still be findable" + ); + } + } + + stop_picker(&shared_picker); +} + +// =================================================================== +// Group 6: Nested directory operations. +// =================================================================== + +/// Files in nested directories should be editable via overlay. +#[test] +fn bigram_overlay_coherence_nested_directory_edits() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + // Verify all initial tokens findable. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for (name, token) in &repo_files { + assert!( + grep_count(picker, token) >= 1, + "initial token {token} ({name}) should be findable" + ); + } + } + + std::thread::sleep(Duration::from_millis(1100)); + + // Edit one file from each subdir group (the realistic repo rotates + // through 8 subdirs, so indices 0,1,2,...,7 cover all dirs). + let mut edited = Vec::new(); + for i in 0..8 { + let (ref name, _) = repo_files[i]; + let token = format!("NESTED_EDIT_{i:02}"); + write_file_with_token(base, name, &token); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(name)); + } + edited.push(token); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for token in &edited { + assert!( + grep_count(picker, token) >= 1, + "nested edit token {token} should be findable" + ); + } + } + + stop_picker(&shared_picker); +} + +// =================================================================== +// Group 7: Fuzzy file search. Verifies that fuzzy filename matching +// works for base files, edited files, overflow files, and after rescan. +// =================================================================== + +/// Helper: run a fuzzy search and return matched file paths. +fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +/// Fuzzy search finds base files, overflow files, and respects deletions. +#[test] +fn bigram_overlay_coherence_fuzzy_search_base_overflow_and_deleted() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + seed_realistic_repo(base); + + // Create files with distinctive names for fuzzy matching ON TOP of + // the realistic repo. + fs::write(base.join("controller_auth.rs"), "auth controller\n").unwrap(); + fs::write(base.join("controller_user.rs"), "user controller\n").unwrap(); + fs::write(base.join("model_invoice.rs"), "invoice model\n").unwrap(); + fs::write(base.join("service_payment.rs"), "payment service\n").unwrap(); + fs::write(base.join("helper_crypto.rs"), "crypto helper\n").unwrap(); + + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + // Fuzzy search for "controller" -- should find both controller files. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let results = fuzzy_search_paths(picker, "controller"); + assert!( + results.len() >= 2, + "fuzzy 'controller' should match at least 2 files, got {results:?}" + ); + assert!( + results.iter().any(|p| p.contains("controller_auth")), + "should find controller_auth.rs" + ); + assert!( + results.iter().any(|p| p.contains("controller_user")), + "should find controller_user.rs" + ); + } + + std::thread::sleep(Duration::from_millis(1100)); + + // Delete controller_user.rs (tombstone). + let user_path = base.join("controller_user.rs"); + fs::remove_file(&user_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&user_path); + } + + // Fuzzy search should no longer return the deleted file. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let results = fuzzy_search_paths(picker, "controller"); + assert!( + results.iter().any(|p| p.contains("controller_auth")), + "should still find controller_auth.rs" + ); + assert!( + !results.iter().any(|p| p.contains("controller_user")), + "deleted controller_user.rs should not appear in fuzzy results" + ); + } + + // Add a new overflow file. + fs::write(base.join("controller_admin.rs"), "admin controller\n").unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join("controller_admin.rs")); + } + + // Fuzzy search should find the new overflow file. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let results = fuzzy_search_paths(picker, "controller"); + assert!( + results.iter().any(|p| p.contains("controller_admin")), + "overflow file controller_admin.rs should appear in fuzzy results, got {results:?}" + ); + assert!( + results.iter().any(|p| p.contains("controller_auth")), + "base file controller_auth.rs should still be in fuzzy results" + ); + } + + stop_picker(&shared_picker); +} + +/// Fuzzy search works after rescan: base files + committed new files all +/// appear, and the deleted files are gone. +#[test] +fn bigram_overlay_coherence_fuzzy_search_after_rescan() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + seed_realistic_repo(base); + + // Add distinctive router/middleware files ON TOP of the realistic repo. + fs::write(base.join("router_api.rs"), "api router\n").unwrap(); + fs::write(base.join("router_web.rs"), "web router\n").unwrap(); + fs::write(base.join("middleware_cors.rs"), "cors middleware\n").unwrap(); + + git_init_and_commit(base); + + let (shared_picker, shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Add a new file and delete an existing one. + fs::write(base.join("router_grpc.rs"), "grpc router\n").unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join("router_grpc.rs")); + } + + let web_path = base.join("router_web.rs"); + fs::remove_file(&web_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&web_path); + } + + // Commit and rescan. + git_add_and_commit(base, "add grpc, remove web"); + + shared_picker + .trigger_full_rescan_async(&shared_frecency) + .expect("rescan should succeed"); + assert!( + shared_picker.wait_for_scan(Duration::from_secs(15)), + "Timed out waiting for scan to complete" + ); + + // After rescan, fuzzy search should reflect the committed state. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let results = fuzzy_search_paths(picker, "router"); + assert!( + results.iter().any(|p| p.contains("router_api")), + "router_api.rs should be in results" + ); + assert!( + results.iter().any(|p| p.contains("router_grpc")), + "committed router_grpc.rs should be in results, got {results:?}" + ); + assert!( + !results.iter().any(|p| p.contains("router_web")), + "deleted router_web.rs should not be in results" + ); + } + + stop_picker(&shared_picker); +} + +/// Combined: fuzzy search + grep on the same overlay state. Exercises +/// both search paths after a mix of edits, creates, and deletes. +#[test] +fn bigram_overlay_coherence_fuzzy_and_grep_combined() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let repo_files = seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + std::thread::sleep(Duration::from_millis(1100)); + + // Edit a base file. + let (ref edit_name, ref _edit_old_token) = repo_files[5]; + write_file_with_token(base, edit_name, "COMBINED_EDIT_TOKEN"); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join(edit_name)); + } + + // Add an overflow file with a distinctive name. + fs::write( + base.join("unique_overflow_widget.rs"), + "OVERFLOW_WIDGET_CONTENT\n", + ) + .unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.handle_create_or_modify(base.join("unique_overflow_widget.rs")); + } + + // Delete a base file. + let (ref del_name, ref del_token) = repo_files[10]; + let del_path = base.join(del_name); + fs::remove_file(&del_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&del_path); + } + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + // Grep: edited content findable. + assert!( + grep_count(picker, "COMBINED_EDIT_TOKEN") >= 1, + "grep should find edited base file" + ); + // Grep: overflow content findable. + assert!( + grep_count(picker, "OVERFLOW_WIDGET_CONTENT") >= 1, + "grep should find overflow file content" + ); + // Grep: deleted file's content gone. + assert_eq!( + grep_count(picker, del_token), + 0, + "grep should not find deleted file content" + ); + + // Fuzzy: overflow file findable by name. + let fuzzy = fuzzy_search_paths(picker, "overflow_widget"); + assert!( + fuzzy.iter().any(|p| p.contains("unique_overflow_widget")), + "fuzzy should find overflow file by name, got {fuzzy:?}" + ); + + // Fuzzy: deleted file not in results. + let fuzzy_del = fuzzy_search_paths(picker, del_name); + let del_file = Path::new(del_name).file_name().unwrap().to_str().unwrap(); + assert!( + !fuzzy_del.iter().any(|p| p.contains(del_file)), + "fuzzy should not find deleted file" + ); + + // Fuzzy: edited file still findable by name. + let fuzzy_edit = fuzzy_search_paths(picker, edit_name); + let edit_file = Path::new(edit_name).file_name().unwrap().to_str().unwrap(); + assert!( + fuzzy_edit.iter().any(|p| p.contains(edit_file)), + "fuzzy should find edited file by name" + ); + } + + stop_picker(&shared_picker); +} + +fn grep_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +fn grep_count(picker: &FilePicker, query: &str) -> usize { + let parsed = parse_grep_query(query); + picker.grep(&parsed, &grep_opts()).matches.len() +} + +/// Wait for scanning to finish (no bigram requirement). +/// Use after `trigger_rescan` which replaces sync_data but does not +/// rebuild the bigram index. +fn wait_for_bigram(shared_picker: &SharedFilePicker) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(50)); + let ready = shared_picker + .read() + .ok() + .map(|guard| { + guard + .as_ref() + .map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } +} + +fn stop_picker(shared_picker: &SharedFilePicker) { + if let Ok(mut guard) = shared_picker.write() { + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + } +} + +fn git_run(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .output() + .unwrap_or_else(|e| panic!("git {:?} failed: {}", args, e)); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git_init_and_commit(dir: &Path) { + git_run(dir, &["init"]); + git_run(dir, &["add", "-A"]); + git_run(dir, &["commit", "-m", "initial"]); +} + +fn git_add_and_commit(dir: &Path, msg: &str) { + git_run(dir, &["add", "-A"]); + git_run(dir, &["commit", "-m", msg]); +} + +fn make_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, // we drive events manually + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + (shared_picker, shared_frecency) +} + +/// Generate a file with a unique token embedded. +fn write_file_with_token(dir: &Path, name: &str, token: &str) { + let content = format!( + "// auto-generated file: {name}\nconst TOKEN: &str = \"{token}\";\nfn main() {{}}\n" + ); + fs::write(dir.join(name), content).unwrap(); +} + +/// Create a realistic 200-file repository with subdirectories and diverse +/// content. Files are split into 4 thematic groups (50 each) so that +/// group-specific bigrams appear in ~25% of files, keeping the bigram +/// index discriminating (columns retained for 3-90% of files). +/// +/// Uses only leaf directories (no files in parent dirs that also have +/// child dirs) so that the directory sort order is consistent with the +/// binary search in `find_file_index`. +/// +/// Returns `Vec<(relative_path, token)>` for all 200 files. +fn seed_realistic_repo(dir: &Path) -> Vec<(String, String)> { + let subdirs = [ + "src/core", + "src/models", + "src/net", + "src/utils", + "tests/integration", + "tests/unit", + "lib/helpers", + "lib/internal", + ]; + for subdir in &subdirs { + fs::create_dir_all(dir.join(subdir)).unwrap(); + } + + let groups: &[(&str, &[&str])] = &[ + ( + "database", + &[ + "SELECT", + "INSERT", + "transaction", + "rollback", + "schema", + "migration", + "postgresql", + "foreign_key", + "primary_key", + "tablespace", + "index_column", + "constraint", + "aggregate", + "partition", + "replication", + ], + ), + ( + "network", + &[ + "socket", + "listen", + "handshake", + "bandwidth", + "latency", + "throughput", + "packet", + "firewall", + "proxy", + "gateway", + "protocol", + "ethernet", + "timeout", + "connection", + "certificate", + ], + ), + ( + "graphics", + &[ + "vertex", + "shader", + "fragment", + "rasterize", + "polygon", + "texture", + "framebuffer", + "quaternion", + "viewport", + "projection", + "tessellation", + "antialiasing", + "voxel", + "billboard", + "raytracing", + ], + ), + ( + "audio", + &[ + "waveform", + "amplitude", + "frequency", + "oscillator", + "synthesizer", + "reverb", + "envelope", + "spectrogram", + "samplerate", + "bitdepth", + "compressor", + "equalizer", + "decibel", + "harmonics", + "chorus", + ], + ), + ]; + + let mut files = Vec::with_capacity(200); + + for (group_idx, (domain, words)) in groups.iter().enumerate() { + for file_in_group in 0..50 { + let idx = group_idx * 50 + file_in_group; + let subdir = subdirs[idx % subdirs.len()]; + let relative_path = format!("{subdir}/repo_{idx:04}.rs"); + let token = format!("REPO_TOKEN_{idx:04}"); + + let w1 = words[file_in_group % words.len()]; + let w2 = words[(file_in_group + 3) % words.len()]; + let w3 = words[(file_in_group + 7) % words.len()]; + let w4 = words[(file_in_group + 11) % words.len()]; + + let content = format!( + "// {domain} module {idx}\n\ + pub fn {w1}_{idx}() -> Result<(), Error> {{\n\ + let ctx = {w2}_context();\n\ + {w3}_process(&ctx)?;\n\ + {w4}_finalize(&ctx)\n\ + }}\n\ + const MARKER: &str = \"{token}\";\n" + ); + fs::write(dir.join(&relative_path), content).unwrap(); + files.push((relative_path, token)); + } + } + + files +} + +fn fuzzy_grep_opts() -> GrepSearchOptions { + GrepSearchOptions { + mode: GrepMode::Fuzzy, + ..grep_opts() + } +} + +fn fuzzy_grep_count(picker: &FilePicker, query: &str) -> usize { + let parsed = parse_grep_query(query); + picker.grep(&parsed, &fuzzy_grep_opts()).matches.len() +} + +/// Overflow files must be searchable via fuzzy grep. +/// +/// The fuzzy grep path uses bigram prefiltering which only covers base +/// files. Overflow files (index >= base_count) must be appended to the +/// search set unconditionally, same as the plain-text grep path does. +/// +/// The realistic repo seeds 200 files split into 4 thematic groups so +/// that group-specific bigrams land in ~25% of files -- right in the +/// retention sweet spot. +#[test] +fn bigram_overlay_coherence_fuzzy_grep_finds_overflow_files() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + seed_realistic_repo(base); + git_init_and_commit(base); + + let (shared_picker, _shared_frecency) = make_picker(base); + wait_for_bigram(&shared_picker); + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + picker.bigram_index().is_some(), + "bigram index must be built" + ); + let idx = picker.bigram_index().unwrap(); + assert!( + idx.columns_used() > 10, + "index should retain enough columns to be discriminating, got {}", + idx.columns_used() + ); + } + + std::thread::sleep(Duration::from_millis(1100)); + + // Add overflow file using "graphics" vocabulary -- its bigrams ("vertex", + // "shader", etc.) appear in ~25% of base files, so the prefilter will + // produce a candidate set that covers group C files but NOT the overflow + // file (its index is beyond the bitset). + let new_path = base.join("overflow_graphics.rs"); + fs::write( + &new_path, + "pub fn vertex_shader_overflow_marker() {\n \ + let mesh = tessellation_pipeline();\n \ + rasterize_fragment(&mesh);\n\ + println!(\"rendered\");\n}\n", + ) + .unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!(picker.handle_create_or_modify(&new_path).is_some()); + assert_eq!(picker.get_overflow_files().len(), 1); + } + + // Plain text grep finds the overflow file (known to work). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + grep_count(picker, "vertex_shader_overflow_marker") >= 1, + "plain text grep should find overflow file content" + ); + } + + // Fuzzy grep: query uses "vertex_shader" -- bigrams "ve","er","rt","te","ex", + // "sh","ha","ad","de","er" overlap with group C base files. The prefilter + // will produce Some(candidates) covering those base files but the overflow + // file's index is beyond the bitset -> silently dropped. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert!( + fuzzy_grep_count(picker, "vertex_shader") >= 1, + "BUG: fuzzy grep should find overflow file content but bigram \ + prefiltering drops overflow files whose index exceeds the \ + candidate bitset length" + ); + } + + stop_picker(&shared_picker); +} diff --git a/crates/fff-core/tests/bigram_overlay_integration.rs b/crates/fff-core/tests/bigram_overlay_integration.rs new file mode 100644 index 0000000..c663acf --- /dev/null +++ b/crates/fff-core/tests/bigram_overlay_integration.rs @@ -0,0 +1,410 @@ +//! Integration test: verify that modifying a file after the bigram index is built +//! still makes the new content findable via grep (through the overlay layer). + +use std::fs; +use std::time::Duration; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency}; + +/// Create a temp directory with some initial files, run the full picker lifecycle, +/// then modify a file and verify grep finds the new content. +#[test] +fn modified_file_findable_via_overlay() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // Create initial files with known content. + fs::write(base.join("alpha.txt"), "hello world\nfoo bar\n").unwrap(); + fs::write( + base.join("beta.txt"), + "some other content\nnothing special\n", + ) + .unwrap(); + fs::write(base.join("gamma.txt"), "yet another file\nmore lines\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, // we drive events manually + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + // Wait for scan + bigram build to complete. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + std::thread::sleep(Duration::from_millis(50)); + + let ready = shared_picker + .read() + .ok() + .map(|guard| { + guard + .as_ref() + .map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for scan + bigram build" + ); + } + + // Sanity check: the 3 files are indexed. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + assert_eq!(picker.get_files().len(), 3, "Expected 3 files after scan"); + assert!( + picker.bigram_index().is_some(), + "Bigram index should be built" + ); + assert!( + picker.bigram_overlay().is_some(), + "Overlay should be initialized" + ); + } + + // "UNIQUE_NEEDLE" should NOT exist in any file yet. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let parsed = parse_grep_query("UNIQUE_NEEDLE"); + let opts = grep_opts(); + let result = picker.grep(&parsed, &opts); + assert_eq!( + result.matches.len(), + 0, + "UNIQUE_NEEDLE should not exist before modification" + ); + } + + // Sleep so the filesystem mtime (seconds granularity) advances past the + // value recorded during scan. Without this, on_create_or_modify skips + // mmap invalidation and grep reads stale cached content. + std::thread::sleep(Duration::from_millis(1100)); + + // Write new content containing the needle. + let modified_path = base.join("beta.txt"); + fs::write( + &modified_path, + "some other content\nUNIQUE_NEEDLE is here\nnothing special\n", + ) + .unwrap(); + + // Simulate watcher event: call on_create_or_modify. + // This updates the overlay's bigrams and invalidates the mmap cache. + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + let result = picker.handle_create_or_modify(&modified_path); + assert!( + result.is_some(), + "on_create_or_modify should return the file" + ); + } + + // The bigram index was built BEFORE the modification, so without the + // overlay, beta.txt would be filtered out (its old bigrams don't contain + // "UNIQUE_NEEDLE"). The overlay should fix that. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let parsed = parse_grep_query("UNIQUE_NEEDLE"); + let opts = grep_opts(); + let result = picker.grep(&parsed, &opts); + assert!( + !result.matches.is_empty(), + "UNIQUE_NEEDLE should be findable after modification (overlay adds the candidate back)" + ); + // May find 1 or 2 matches depending on mmap cache state — the important + // thing is that the modified content IS found. + assert!( + result + .matches + .iter() + .any(|m| m.line_content.contains("UNIQUE_NEEDLE")), + "At least one match should contain UNIQUE_NEEDLE" + ); + } + + // Cleanup: stop background watcher. + if let Ok(mut guard) = shared_picker.write() { + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + } +} + +/// Verify that deleting a file makes its content un-findable via grep. +#[test] +fn deleted_file_excluded_via_overlay() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + fs::write(base.join("keep.txt"), "keep this content\n").unwrap(); + fs::write(base.join("remove.txt"), "DELETEME_TOKEN is here\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, // we drive events manually + ..Default::default() + }, + ) + .unwrap(); + + wait_for_bigram(&shared_picker); + + // Sanity: DELETEME_TOKEN is findable. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let result = grep_for(picker, "DELETEME_TOKEN"); + assert_eq!( + result.matches.len(), + 1, + "Token should be found before delete" + ); + } + + // Delete the file on disk and via picker. + let remove_path = base.join("remove.txt"); + fs::remove_file(&remove_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.remove_file_by_path(&remove_path), + "remove should succeed" + ); + } + + // Token should no longer be found (tombstone in overlay clears the candidate). + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let result = grep_for(picker, "DELETEME_TOKEN"); + assert_eq!( + result.matches.len(), + 0, + "DELETEME_TOKEN should not be found after deletion (tombstone in overlay)" + ); + } + + if let Ok(mut guard) = shared_picker.write() { + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + } +} + +/// Verify that a newly added file (in overflow) is findable via grep. +#[test] +fn new_file_findable_after_add() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + fs::write(base.join("existing.txt"), "original content\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, // we drive events manually + ..Default::default() + }, + ) + .unwrap(); + + wait_for_bigram(&shared_picker); + + // Create a new file on disk after the index was built. + let new_path = base.join("newcomer.txt"); + fs::write(&new_path, "BRAND_NEW_TOKEN lives here\n").unwrap(); + + // Simulate watcher detecting the new file. + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + let result = picker.handle_create_or_modify(&new_path); + assert!( + result.is_some(), + "on_create_or_modify should return the new file" + ); + } + + // The new file is in overflow, not in the base files slice. + // grep_search currently only searches base files, so we need to verify + // the overflow file is accessible. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let overflow = picker.get_overflow_files(); + assert_eq!(overflow.len(), 1, "Should have 1 overflow file"); + assert!( + overflow[0].relative_path(picker).ends_with("newcomer.txt"), + "Overflow file should be newcomer.txt" + ); + } + + if let Ok(mut guard) = shared_picker.write() { + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + } +} + +/// Verify that a file modified after index build is findable via regex grep +/// through the overlay. This catches a regression where `extract_bigrams` on +/// the raw regex string (e.g. "NEEDLE.*HERE") produces bogus bigrams containing +/// `.` and `*`, causing `query_modified` to miss the file. +#[test] +fn modified_file_findable_via_regex_overlay() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + fs::write(base.join("alpha.txt"), "hello world\nfoo bar\n").unwrap(); + fs::write( + base.join("beta.txt"), + "some other content\nnothing special\n", + ) + .unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, // we drive events manually + ..Default::default() + }, + ) + .unwrap(); + + wait_for_bigram(&shared_picker); + + // Advance mtime past the scan timestamp so the cache is invalidated. + std::thread::sleep(Duration::from_millis(1100)); + + // Write content that matches the regex "NEEDLE.*HERE" into beta.txt. + let modified_path = base.join("beta.txt"); + fs::write( + &modified_path, + "some other content\nNEEDLE is right HERE\nnothing special\n", + ) + .unwrap(); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!(picker.handle_create_or_modify(&modified_path).is_some()); + } + + // Regex grep should find the modified file through the overlay. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let parsed = parse_grep_query("NEEDLE.*HERE"); + let opts = GrepSearchOptions { + mode: GrepMode::Regex, + ..grep_opts() + }; + let result = picker.grep(&parsed, &opts); + assert!( + !result.matches.is_empty(), + "Regex grep should find NEEDLE.*HERE in modified file via overlay" + ); + assert!(result.matches[0].line_content.contains("NEEDLE")); + } + + if let Ok(mut guard) = shared_picker.write() { + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + } +} + +// ── Helpers ───────────────────────────────────────────────────────────── + +fn grep_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +fn grep_for<'a>(picker: &'a FilePicker, query: &str) -> fff_search::grep::GrepResult<'a> { + let parsed = parse_grep_query(query); + picker.grep(&parsed, &grep_opts()) +} + +fn wait_for_bigram(shared_picker: &SharedFilePicker) { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + std::thread::sleep(Duration::from_millis(50)); + let ready = shared_picker + .read() + .ok() + .map(|guard| { + guard + .as_ref() + .map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } +} diff --git a/crates/fff-core/tests/drop_during_post_scan.rs b/crates/fff-core/tests/drop_during_post_scan.rs new file mode 100644 index 0000000..f69090f --- /dev/null +++ b/crates/fff-core/tests/drop_during_post_scan.rs @@ -0,0 +1,161 @@ +// Regression pinning: dropping a picker during poset scan off-lock time +use std::fs; +use std::path::Path; +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions}; +use fff_search::{FilePickerOptions, QueryParser, SharedFilePicker, SharedFrecency}; + +fn seed_files(dir: &Path, count: usize) { + for i in 0..count { + let subdir = dir.join(format!("dir_{}", i / 20)); + fs::create_dir_all(&subdir).unwrap(); + fs::write( + subdir.join(format!("file_{i}.rs")), + format!("pub fn func_{i}() {{ /* token_{i} */ }}\n"), + ) + .unwrap(); + } +} + +fn git_init(dir: &Path) { + let run = |args: &[&str]| { + Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .unwrap(); + }; + run(&["init"]); + run(&["add", "-A"]); + run(&["commit", "-m", "init"]); +} + +fn make_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) { + let sp = SharedFilePicker::default(); + let sf = SharedFrecency::default(); + FilePicker::new_with_shared_state( + sp.clone(), + sf.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("init"); + (sp, sf) +} + +/// Drop picker immediately after scan starts — scan thread will find +/// the picker gone and exit cleanly. +#[test] +fn drop_picker_during_walk_no_segfault() { + let tmp = TempDir::new().unwrap(); + seed_files(tmp.path(), 500); + git_init(tmp.path()); + + let (sp, _sf) = make_picker(tmp.path()); + // Don't wait — drop immediately while walk is likely in progress + drop(sp); + + // If we get here without SIGSEGV, the test passes. + std::thread::sleep(Duration::from_millis(200)); +} + +/// Drop picker while post-scan indexing is running. The snapshot holds +/// Arc clones that keep the buffers alive. +#[test] +fn drop_picker_during_post_scan_no_segfault() { + let tmp = TempDir::new().unwrap(); + seed_files(tmp.path(), 500); + git_init(tmp.path()); + + let (sp, _sf) = make_picker(tmp.path()); + + // Wait for walk to finish (files are searchable) but post-scan is + // still running (bigram not yet built). + sp.wait_for_scan(Duration::from_secs(10)); + + // At this point post_scan_indexing_active is likely true. + // Drop the picker — this releases the picker's Arc clones, but the + // post-scan snapshot's clones keep the buffers alive. + if let Ok(mut guard) = sp.write() { + guard.take(); // drop the FilePicker + } + + // Give post-scan threads time to run against the "dead" picker. + // They must not segfault. + std::thread::sleep(Duration::from_secs(2)); +} + +/// Drop picker from a second thread while the first thread is doing +/// fuzzy searches. Verifies no segfault from interleaved access. +#[test] +fn drop_picker_concurrent_with_search_no_segfault() { + let tmp = TempDir::new().unwrap(); + seed_files(tmp.path(), 500); + git_init(tmp.path()); + + let (sp, _sf) = make_picker(tmp.path()); + sp.wait_for_scan(Duration::from_secs(10)); + + let sp_clone = sp.clone(); + let running = Arc::new(AtomicBool::new(true)); + let running_clone = running.clone(); + + // Searcher thread: continuously queries while the picker lives + let searcher = std::thread::spawn(move || { + let parser = QueryParser::default(); + while running_clone.load(Ordering::Relaxed) { + if let Ok(guard) = sp_clone.read() { + if let Some(picker) = guard.as_ref() { + let query = parser.parse("func"); + let _ = picker.fuzzy_search(&query, None, FuzzySearchOptions::default()); + } + } + std::thread::sleep(Duration::from_millis(1)); + } + }); + + // Let searches run for a bit, then drop + std::thread::sleep(Duration::from_millis(100)); + if let Ok(mut guard) = sp.write() { + guard.take(); + } + std::thread::sleep(Duration::from_millis(100)); + + running.store(false, Ordering::Relaxed); + searcher.join().unwrap(); +} + +/// Repeated init + wait + clean-drop cycle. This is the pattern that +/// SIGSEGV'd on the pre-refactor code in the benchmark. +#[test] +fn repeated_init_and_drop_no_segfault() { + let tmp = TempDir::new().unwrap(); + seed_files(tmp.path(), 200); + git_init(tmp.path()); + + for _ in 0..5 { + let (sp, _sf) = make_picker(tmp.path()); + sp.wait_for_scan(Duration::from_secs(10)); + sp.wait_for_indexing_complete(Duration::from_secs(30)); + if let Ok(mut guard) = sp.write() + && let Some(mut picker) = guard.take() + { + picker.stop_background_monitor(); + } + } +} diff --git a/crates/fff-core/tests/fixtures/binaries/codex_view b/crates/fff-core/tests/fixtures/binaries/codex_view new file mode 100644 index 0000000..1e5a6cc Binary files /dev/null and b/crates/fff-core/tests/fixtures/binaries/codex_view differ diff --git a/crates/fff-core/tests/fixtures/binaries/codex_view.codex b/crates/fff-core/tests/fixtures/binaries/codex_view.codex new file mode 100644 index 0000000..54725b3 Binary files /dev/null and b/crates/fff-core/tests/fixtures/binaries/codex_view.codex differ diff --git a/crates/fff-core/tests/fs_delete_handler_test.rs b/crates/fff-core/tests/fs_delete_handler_test.rs new file mode 100644 index 0000000..a37e443 --- /dev/null +++ b/crates/fff-core/tests/fs_delete_handler_test.rs @@ -0,0 +1,320 @@ +//! Reproducer: macOS FSEvents does not deliver Remove events for files +//! deleted from NonRecursive-watched directories when multiple directories +//! are watched via stop/restart cycles. +//! +//! This test watches a temp directory NonRecursively, creates a file, +//! verifies the Create event, deletes the file, and checks whether a +//! Remove (or any) event is delivered. + +use notify::event::*; +use notify::{Config, EventKindMask, RecommendedWatcher, RecursiveMode, Watcher}; +use std::fs; +use std::path::PathBuf; +use std::sync::mpsc; +use std::time::Duration; + +fn setup_temp_git_repo() -> (PathBuf, tempfile::TempDir) { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().canonicalize().unwrap(); + + // Create a git repo like the bun test does + std::process::Command::new("git") + .args(["init", "-b", "main"]) + .current_dir(&dir) + .output() + .unwrap(); + + fs::write(dir.join("hello.txt"), "hello\n").unwrap(); + fs::create_dir_all(dir.join("src")).unwrap(); + fs::write(dir.join("src/main.rs"), "fn main() {}\n").unwrap(); + + std::process::Command::new("git") + .args(["add", "-A"]) + .current_dir(&dir) + .output() + .unwrap(); + std::process::Command::new("git") + .args(["commit", "-m", "init"]) + .current_dir(&dir) + .output() + .unwrap(); + + (dir, tmp) +} + +/// Raw notify watcher: single NonRecursive watch on a directory. +/// Create a file, delete it, check if Remove event is delivered. +#[test] +fn raw_notify_nonrecursive_detects_deletion() { + let (dir, _tmp) = setup_temp_git_repo(); + let (tx, rx) = mpsc::channel(); + + let config = Config::default() + .with_follow_symlinks(false) + .with_event_kinds(EventKindMask::CORE); + + let mut watcher = RecommendedWatcher::new( + move |res: notify::Result| { + if let Ok(ev) = res { + let _ = tx.send(ev); + } + }, + config, + ) + .unwrap(); + + // Watch ONLY the root dir NonRecursively (like fff does) + watcher + .watch(dir.as_path(), RecursiveMode::NonRecursive) + .unwrap(); + + // Let the watcher stabilize + std::thread::sleep(Duration::from_millis(500)); + + // Drain any startup events + while rx.try_recv().is_ok() {} + + // Create a file + let file_path = dir.join("testfile.txt"); + fs::write(&file_path, "content\n").unwrap(); + + // Wait for create event + let mut got_create = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + eprintln!(" [create phase] event: {:?} paths={:?}", ev.kind, ev.paths); + if matches!(ev.kind, EventKind::Create(_)) && ev.paths.contains(&file_path) { + got_create = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + assert!(got_create, "Expected Create event for testfile.txt"); + + // Drain remaining events from the create + std::thread::sleep(Duration::from_millis(300)); + while rx.try_recv().is_ok() {} + + // Delete the file + fs::remove_file(&file_path).unwrap(); + eprintln!(" File deleted: {}", file_path.display()); + + // Wait for any event related to the deletion + let mut got_removal_event = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + eprintln!(" [delete phase] event: {:?} paths={:?}", ev.kind, ev.paths); + if ev.paths.contains(&file_path) { + got_removal_event = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + + assert!( + got_removal_event, + "Expected some event for deleted testfile.txt but got none within 5s" + ); +} + +/// Same test but with MULTIPLE NonRecursive watches (base + src + .git) +/// to match what fff actually does. Each watch() call stops/restarts the FSEvents stream. +#[test] +fn raw_notify_multi_nonrecursive_detects_deletion() { + let (dir, _tmp) = setup_temp_git_repo(); + let (tx, rx) = mpsc::channel(); + + let config = Config::default() + .with_follow_symlinks(false) + .with_event_kinds(EventKindMask::CORE); + + let mut watcher = RecommendedWatcher::new( + move |res: notify::Result| { + if let Ok(ev) = res { + let _ = tx.send(ev); + } + }, + config, + ) + .unwrap(); + + // Watch multiple directories NonRecursively — EACH call restarts the FSEvents stream + watcher + .watch(dir.as_path(), RecursiveMode::NonRecursive) + .unwrap(); + watcher + .watch(&dir.join("src"), RecursiveMode::NonRecursive) + .unwrap(); + watcher + .watch(&dir.join(".git"), RecursiveMode::NonRecursive) + .unwrap(); + + // Let the watcher stabilize + std::thread::sleep(Duration::from_millis(500)); + while rx.try_recv().is_ok() {} + + // Create a file in root dir + let file_path = dir.join("testfile.txt"); + fs::write(&file_path, "content\n").unwrap(); + + // Wait for create event + let mut got_create = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + eprintln!(" [multi-create] event: {:?} paths={:?}", ev.kind, ev.paths); + if matches!(ev.kind, EventKind::Create(_)) && ev.paths.contains(&file_path) { + got_create = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + assert!( + got_create, + "Expected Create event for testfile.txt with multi-watch" + ); + + // Drain + std::thread::sleep(Duration::from_millis(300)); + while rx.try_recv().is_ok() {} + + // Delete the file + fs::remove_file(&file_path).unwrap(); + eprintln!(" File deleted: {}", file_path.display()); + + // Wait for any event related to the deletion + let mut got_removal_event = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + eprintln!(" [multi-delete] event: {:?} paths={:?}", ev.kind, ev.paths); + if ev.paths.contains(&file_path) { + got_removal_event = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + + assert!( + got_removal_event, + "Expected some event for deleted testfile.txt with multi-watch but got none within 5s" + ); +} + +/// Test with debouncer (matching exactly what fff uses) +#[test] +fn debounced_nonrecursive_detects_deletion() { + use notify_debouncer_full::{DebounceEventResult, NoCache, new_debouncer_opt}; + + let (dir, _tmp) = setup_temp_git_repo(); + let (tx, rx) = mpsc::channel(); + + let config = Config::default() + .with_follow_symlinks(false) + .with_event_kinds(EventKindMask::CORE); + + let mut debouncer: notify_debouncer_full::Debouncer = + new_debouncer_opt( + Duration::from_millis(250), + Some(Duration::from_millis(125)), + move |result: DebounceEventResult| { + if let Ok(events) = result { + for ev in events { + eprintln!( + " [debounced-cb] kind={:?} paths={:?}", + ev.event.kind, ev.event.paths + ); + let _ = tx.send(ev); + } + } + }, + NoCache::new(), + config, + ) + .unwrap(); + + // Watch like fff does + debouncer + .watch(dir.as_path(), RecursiveMode::NonRecursive) + .unwrap(); + debouncer + .watch(&dir.join("src"), RecursiveMode::NonRecursive) + .unwrap(); + debouncer + .watch(&dir.join(".git"), RecursiveMode::NonRecursive) + .unwrap(); + + // Longer stabilization — each watch() restarts the FSEvents stream + std::thread::sleep(Duration::from_secs(1)); + while rx.try_recv().is_ok() {} + + // Create file + let file_path = dir.join("testfile.txt"); + fs::write(&file_path, "content\n").unwrap(); + + let mut got_create = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + if ev.event.paths.contains(&file_path) { + got_create = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + assert!(got_create, "Expected Create event via debouncer"); + + // Wait for debounce to fully flush + std::thread::sleep(Duration::from_millis(500)); + while rx.try_recv().is_ok() {} + + // Delete + fs::remove_file(&file_path).unwrap(); + eprintln!(" File deleted: {}", file_path.display()); + + // Wait for ANY event for this path + let mut got_event = false; + let mut event_kind = String::new(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(ev) => { + if ev.event.paths.contains(&file_path) { + event_kind = format!("{:?}", ev.event.kind); + got_event = true; + break; + } + } + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(_) => break, + } + } + + assert!( + got_event, + "Expected some event for deleted testfile.txt via debouncer but got none within 5s" + ); + eprintln!(" Got event kind: {}", event_kind); +} diff --git a/crates/fff-core/tests/fuzz_file_operations.rs b/crates/fff-core/tests/fuzz_file_operations.rs new file mode 100644 index 0000000..8468493 --- /dev/null +++ b/crates/fff-core/tests/fuzz_file_operations.rs @@ -0,0 +1,854 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tempfile::TempDir; + +use rand::rngs::SmallRng; +use rand::{RngCore, SeedableRng}; + +use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{ + FilePickerOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency, +}; + +const DOMAINS: &[&str] = &[ + r#" +use std::net::{TcpStream, SocketAddr}; +fn establish_connection(addr: SocketAddr) -> Result { + let stream = TcpStream::connect(addr)?; + stream.set_nodelay(true)?; + Ok(stream) +} +fn parse_http_header(raw: &[u8]) -> Option<(&str, &str)> { + let line = std::str::from_utf8(raw).ok()?; + let (key, val) = line.split_once(':')?; + Some((key.trim(), val.trim())) +} +"#, + r#" +use sqlx::{PgPool, Row}; +async fn query_users(pool: &PgPool, limit: i64) -> Vec { + sqlx::query("SELECT name FROM users ORDER BY created_at DESC LIMIT $1") + .bind(limit) + .fetch_all(pool) + .await + .unwrap() + .iter() + .map(|row| row.get("name")) + .collect() +} +async fn insert_record(pool: &PgPool, name: &str) -> i64 { + sqlx::query_scalar("INSERT INTO records (name) VALUES ($1) RETURNING id") + .bind(name) + .fetch_one(pool) + .await + .unwrap() +} +"#, + r#" +fn verify_jwt_token(token: &str, secret: &[u8]) -> Result { + let parts: Vec<&str> = token.splitn(3, '.').collect(); + if parts.len() != 3 { return Err(AuthError::MalformedToken); } + let payload = base64_decode(parts[1])?; + let signature = hmac_sha256(secret, &format!("{}.{}", parts[0], parts[1])); + if signature != base64_decode(parts[2])? { return Err(AuthError::InvalidSignature); } + serde_json::from_slice(&payload).map_err(AuthError::Deserialize) +} +fn hash_password(password: &str, salt: &[u8]) -> Vec { + argon2::hash_encoded(password.as_bytes(), salt, &argon2::Config::default()) + .unwrap().into_bytes() +} +"#, + r#" +struct Renderer { framebuffer: Vec, width: usize, height: usize } +impl Renderer { + fn clear(&mut self, color: u32) { self.framebuffer.fill(color); } + fn draw_pixel(&mut self, x: usize, y: usize, color: u32) { + if x < self.width && y < self.height { + self.framebuffer[y * self.width + x] = color; + } + } + fn draw_line(&mut self, x0: i32, y0: i32, x1: i32, y1: i32, color: u32) { + let dx = (x1 - x0).abs(); let dy = -(y1 - y0).abs(); + let mut err = dx + dy; + let (mut cx, mut cy) = (x0, y0); + loop { + self.draw_pixel(cx as usize, cy as usize, color); + if cx == x1 && cy == y1 { break; } + let e2 = 2 * err; + if e2 >= dy { err += dy; cx += if x0 < x1 { 1 } else { -1 }; } + if e2 <= dx { err += dx; cy += if y0 < y1 { 1 } else { -1 }; } + } + } +} +"#, + r#" +use serde::{Serialize, Deserialize}; +#[derive(Serialize, Deserialize)] +struct ConfigFile { log_level: String, max_retries: u32, timeout_ms: u64 } +fn load_config(path: &std::path::Path) -> Result> { + let contents = std::fs::read_to_string(path)?; + let config: ConfigFile = toml::from_str(&contents)?; + Ok(config) +} +fn merge_configs(base: ConfigFile, overlay: ConfigFile) -> ConfigFile { + ConfigFile { + log_level: if overlay.log_level.is_empty() { base.log_level } else { overlay.log_level }, + max_retries: overlay.max_retries.max(base.max_retries), + timeout_ms: overlay.timeout_ms.max(base.timeout_ms), + } +} +"#, + r#" +struct PhysicsBody { position: [f64; 3], velocity: [f64; 3], mass: f64 } +fn apply_gravity(bodies: &mut [PhysicsBody], dt: f64) { + let gravity_constant = 6.674e-11; + let len = bodies.len(); + let mut forces = vec![[0.0f64; 3]; len]; + for i in 0..len { + for j in (i+1)..len { + let dx = bodies[j].position[0] - bodies[i].position[0]; + let dy = bodies[j].position[1] - bodies[i].position[1]; + let dz = bodies[j].position[2] - bodies[i].position[2]; + let dist_sq = dx*dx + dy*dy + dz*dz; + let force_mag = gravity_constant * bodies[i].mass * bodies[j].mass / dist_sq; + let dist = dist_sq.sqrt(); + for k in 0..3 { + let f = force_mag * [dx, dy, dz][k] / dist; + forces[i][k] += f; forces[j][k] -= f; + } + } + } + for (body, force) in bodies.iter_mut().zip(forces.iter()) { + for k in 0..3 { + body.velocity[k] += force[k] / body.mass * dt; + body.position[k] += body.velocity[k] * dt; + } + } +} +"#, + r#" +use std::collections::BTreeMap; +struct CacheEntry { value: V, frequency: u64, last_access: u64 } +struct LFUCache { map: BTreeMap>, capacity: usize, clock: u64 } +impl LFUCache { + fn new(capacity: usize) -> Self { Self { map: BTreeMap::new(), capacity, clock: 0 } } + fn get(&mut self, key: &K) -> Option<&V> { + self.clock += 1; + let entry = self.map.get_mut(key)?; + entry.frequency += 1; + entry.last_access = self.clock; + Some(&entry.value) + } + fn insert(&mut self, key: K, value: V) { + self.clock += 1; + if self.map.len() >= self.capacity { self.evict(); } + self.map.insert(key, CacheEntry { value, frequency: 1, last_access: self.clock }); + } + fn evict(&mut self) { + if let Some(victim) = self.map.keys().min_by_key(|k| { + let e = &self.map[*k]; (e.frequency, e.last_access) + }).cloned() { self.map.remove(&victim); } + } +} +"#, + r#" +fn tokenize_expression(input: &str) -> Vec { + let mut tokens = Vec::new(); + let mut chars = input.chars().peekable(); + while let Some(&ch) = chars.peek() { + match ch { + '0'..='9' => { + let mut num = String::new(); + while let Some(&d) = chars.peek() { + if d.is_ascii_digit() || d == '.' { num.push(d); chars.next(); } + else { break; } + } + tokens.push(Token::Number(num.parse().unwrap())); + } + '+' => { tokens.push(Token::Plus); chars.next(); } + '-' => { tokens.push(Token::Minus); chars.next(); } + '*' => { tokens.push(Token::Star); chars.next(); } + '/' => { tokens.push(Token::Slash); chars.next(); } + '(' => { tokens.push(Token::LParen); chars.next(); } + ')' => { tokens.push(Token::RParen); chars.next(); } + _ if ch.is_whitespace() => { chars.next(); } + _ => { chars.next(); } + } + } + tokens +} +"#, + r#" +use std::sync::mpsc; +use std::thread; +fn parallel_map( + items: Vec, num_threads: usize, f: fn(T) -> R +) -> Vec { + let chunk_size = (items.len() + num_threads - 1) / num_threads; + let (tx, rx) = mpsc::channel(); + let mut handles = Vec::new(); + for (chunk_idx, chunk) in items.into_iter().collect::>() + .chunks(chunk_size).enumerate() + { + let tx = tx.clone(); + let chunk = chunk.to_vec(); + handles.push(thread::spawn(move || { + for (i, item) in chunk.into_iter().enumerate() { + tx.send((chunk_idx * chunk_size + i, f(item))).unwrap(); + } + })); + } + drop(tx); + let mut results: Vec> = vec![None; handles.len() * chunk_size]; + for (idx, result) in rx { if idx < results.len() { results[idx] = Some(result); } } + for h in handles { h.join().unwrap(); } + results.into_iter().flatten().collect() +} +"#, + r#" +struct Compressor { window: Vec, window_size: usize } +impl Compressor { + fn new(window_size: usize) -> Self { + Self { window: Vec::with_capacity(window_size), window_size } + } + fn find_longest_match(&self, data: &[u8], pos: usize) -> (usize, usize) { + let mut best_offset = 0; let mut best_length = 0; + let start = pos.saturating_sub(self.window_size); + for offset in start..pos { + let mut length = 0; + while pos + length < data.len() + && data[offset + length] == data[pos + length] + && length < 258 + { length += 1; } + if length > best_length { best_offset = pos - offset; best_length = length; } + } + (best_offset, best_length) + } + fn compress(&mut self, data: &[u8]) -> Vec { + let mut output = Vec::new(); + let mut pos = 0; + while pos < data.len() { + let (offset, length) = self.find_longest_match(data, pos); + if length >= 3 { + output.push(1); output.extend_from_slice(&(offset as u16).to_le_bytes()); + output.push(length as u8); pos += length; + } else { output.push(0); output.push(data[pos]); pos += 1; } + } + output + } +} +"#, +]; + +struct FileState { + name: String, + token: String, + #[allow(dead_code)] + is_base: bool, + /// Epoch second when this file was last written (used to detect same-second + /// re-edits that wouldn't bump mtime and thus wouldn't invalidate the mmap). + last_write_sec: u64, +} + +#[test] +fn fuzz_file_operations_stress() { + const SEED: u64 = 0xDEAD_BEEF_CAFE_1234; + const INITIAL_FILE_COUNT: usize = 40; + const NUM_ROUNDS: usize = 20; + + let mut rng = SmallRng::seed_from_u64(SEED); + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // Timing accumulators. + let mut t_sleep = Duration::ZERO; + let mut t_git = Duration::ZERO; + let mut t_bigram_wait = Duration::ZERO; + let mut t_grep_plain = Duration::ZERO; + let mut t_grep_regex = Duration::ZERO; + let mut t_fuzzy = Duration::ZERO; + let mut t_dead_check = Duration::ZERO; + let grep_plain_calls = std::sync::atomic::AtomicUsize::new(0); + let grep_regex_calls = std::sync::atomic::AtomicUsize::new(0); + let fuzzy_calls = std::sync::atomic::AtomicUsize::new(0); + let dead_calls = std::sync::atomic::AtomicUsize::new(0); + let test_start = std::time::Instant::now(); + + let mut live_files: Vec = Vec::with_capacity(INITIAL_FILE_COUNT + NUM_ROUNDS); + let mut dead_tokens: Vec = Vec::new(); + let mut next_file_id: usize = 0; + + for i in 0..INITIAL_FILE_COUNT { + let name = format!("seed_{i:04}.rs"); + let token = format!("FUZZ_SEED_{i:04}"); + write_diverse_file(base, &name, &token, i); + live_files.push(FileState { + name, + token, + is_base: true, + last_write_sec: 0, // set before index build, doesn't matter + }); + next_file_id += 1; + } + + let t0 = std::time::Instant::now(); + git_init_and_commit(base); + t_git += t0.elapsed(); + + let shared_picker = SharedFilePicker::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + SharedFrecency::noop(), + FilePickerOptions { + watch: false, // we do not need the backgrodun monitor + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let t0 = std::time::Instant::now(); + wait_for_bigram(&shared_picker); + t_bigram_wait += t0.elapsed(); + + // Sanity: all initial tokens findable via plain grep. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for fs in &live_files { + assert!( + grep_plain_count(picker, &fs.token) >= 1, + "initial sanity: plain grep should find token {} in {}", + fs.token, + fs.name + ); + } + } + + // Sleep so mtime advances past the scan snapshot timestamp. + let t0 = std::time::Instant::now(); + std::thread::sleep(Duration::from_millis(1100)); + t_sleep += t0.elapsed(); + + let mut op_counter: usize = 0; + + for round in 0..NUM_ROUNDS { + let roll: u32 = rng.next_u32() % 100; + + if roll < 40 && !live_files.is_empty() { + // ── EDIT existing file (40%) ── + let idx = rng.next_u32() as usize % live_files.len(); + + // on_create_or_modify uses mtime (seconds granularity) to decide + // whether to invalidate the mmap cache. If we re-edit a file in + // the same second it was last written, the mtime won't change and + // the stale cached content will be returned. Sleep to advance mtime. + let now_sec = epoch_secs(); + if live_files[idx].last_write_sec >= now_sec { + let t0 = std::time::Instant::now(); + std::thread::sleep(Duration::from_millis(1100)); + t_sleep += t0.elapsed(); + } + + let old_token = live_files[idx].token.clone(); + let new_token = format!("FUZZ_{round:02}_{op_counter:04}"); + let name = &live_files[idx].name; + let domain_idx = rng.next_u32() as usize % DOMAINS.len(); + write_diverse_file_with_domain(base, name, &new_token, domain_idx); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.handle_create_or_modify(base.join(name)).is_some(), + "round {round}: on_create_or_modify({name}) should succeed for edit" + ); + } + + dead_tokens.push(old_token); + live_files[idx].token = new_token; + live_files[idx].last_write_sec = epoch_secs(); + op_counter += 1; + } else if roll < 60 { + // ── CREATE new file (20%) ── + let name = format!("created_{next_file_id:04}.rs"); + let token = format!("FUZZ_{round:02}_{op_counter:04}"); + let domain_idx = rng.next_u32() as usize % DOMAINS.len(); + write_diverse_file_with_domain(base, &name, &token, domain_idx); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.handle_create_or_modify(base.join(&name)).is_some(), + "round {round}: on_create_or_modify({name}) should succeed for create" + ); + } + + live_files.push(FileState { + name, + token, + is_base: false, + last_write_sec: epoch_secs(), + }); + next_file_id += 1; + op_counter += 1; + } else if roll < 75 && !live_files.is_empty() { + // ── DELETE existing file (15%) ── + let idx = rng.next_u32() as usize % live_files.len(); + let removed = live_files.swap_remove(idx); + let path = base.join(&removed.name); + fs::remove_file(&path).unwrap(); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.remove_file_by_path(&path), + "round {round}: remove_file_by_path({}) should succeed", + removed.name + ); + } + + dead_tokens.push(removed.token); + op_counter += 1; + } else if roll < 85 && !live_files.is_empty() { + // ── RENAME file (10%) ── + let idx = rng.next_u32() as usize % live_files.len(); + let old_name = live_files[idx].name.clone(); + let old_path = base.join(&old_name); + let content = fs::read_to_string(&old_path).unwrap(); + + // Remove old file from disk + picker. + fs::remove_file(&old_path).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + picker.remove_file_by_path(&old_path); + } + + // Create new file with same content but different name. + let new_name = format!("renamed_{next_file_id:04}.rs"); + fs::write(base.join(&new_name), &content).unwrap(); + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker + .handle_create_or_modify(base.join(&new_name)) + .is_some(), + "round {round}: on_create_or_modify({new_name}) should succeed for rename" + ); + } + + live_files[idx].name = new_name; + live_files[idx].is_base = false; + live_files[idx].last_write_sec = epoch_secs(); + next_file_id += 1; + op_counter += 1; + } + // else: no-op / read-only (15%) — just run verification below. + + // ── VERIFY after every round ── + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + for fs in &live_files { + // Plain text grep: every live token must be found. + let t0 = std::time::Instant::now(); + let plain_count = grep_plain_count(picker, &fs.token); + t_grep_plain += t0.elapsed(); + grep_plain_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert!( + plain_count >= 1, + "round {round}: plain grep should find live token {} in {} (got {plain_count})", + fs.token, + fs.name + ); + + // Regex grep: search with `{first5}.*{last5}` pattern. + let regex_pattern = build_regex_pattern(&fs.token); + let t0 = std::time::Instant::now(); + let regex_count = grep_regex_count(picker, ®ex_pattern); + t_grep_regex += t0.elapsed(); + grep_regex_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert!( + regex_count >= 1, + "round {round}: regex grep '{}' should find live token {} in {} (got {regex_count})", + regex_pattern, + fs.token, + fs.name + ); + + // Fuzzy file search: every live file must be findable by name. + let stem = extract_stem(&fs.name); + let t0 = std::time::Instant::now(); + let fuzzy_results = fuzzy_search_paths(picker, &stem); + t_fuzzy += t0.elapsed(); + fuzzy_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert!( + fuzzy_results.iter().any(|p| p.contains(&fs.name)), + "round {round}: fuzzy search '{}' should find file {} in results: {:?}", + stem, + fs.name, + fuzzy_results + ); + } + + // Dead tokens must return 0 grep results. + for dead in &dead_tokens { + let t0 = std::time::Instant::now(); + let count = grep_plain_count(picker, dead); + t_dead_check += t0.elapsed(); + dead_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert_eq!( + count, 0, + "round {round}: dead token {dead} should NOT be findable (got {count})" + ); + } + } + } + + let total = test_start.elapsed(); + let t_overhead = t_sleep + t_bigram_wait + t_git; + let t_search = t_grep_plain + t_grep_regex + t_fuzzy + t_dead_check; + let t_mutations = total.saturating_sub(t_overhead + t_search); + let n_grep_plain = grep_plain_calls.load(std::sync::atomic::Ordering::Relaxed); + let n_grep_regex = grep_regex_calls.load(std::sync::atomic::Ordering::Relaxed); + let n_fuzzy = fuzzy_calls.load(std::sync::atomic::Ordering::Relaxed); + let n_dead = dead_calls.load(std::sync::atomic::Ordering::Relaxed); + eprintln!("\n╔══════════════════════════════════════════════════════╗"); + eprintln!("║ Fuzz Test Performance Breakdown ║"); + eprintln!("╠══════════════════════════════════════════════════════╣"); + eprintln!( + "║ Total wall time: {:>8.1}ms ║", + total.as_secs_f64() * 1000.0 + ); + eprintln!("║ ── Overhead ─────────────────────────────────────── ║"); + eprintln!( + "║ Sleep (mtime waits): {:>8.1}ms ║", + t_sleep.as_secs_f64() * 1000.0 + ); + eprintln!( + "║ Git init+commit: {:>8.1}ms ║", + t_git.as_secs_f64() * 1000.0 + ); + eprintln!( + "║ Bigram index build+scan: {:>8.1}ms ║", + t_bigram_wait.as_secs_f64() * 1000.0 + ); + eprintln!( + "║ ── Search ({:>3} live files, {:>3} dead tokens) ────── ║", + live_files.len(), + dead_tokens.len() + ); + eprintln!( + "║ Plain grep: {:>4} calls {:>8.1}ms ({:>6.1}µs/call) ║", + n_grep_plain, + t_grep_plain.as_secs_f64() * 1000.0, + t_grep_plain.as_secs_f64() * 1_000_000.0 / n_grep_plain.max(1) as f64 + ); + eprintln!( + "║ Regex grep: {:>4} calls {:>8.1}ms ({:>6.1}µs/call) ║", + n_grep_regex, + t_grep_regex.as_secs_f64() * 1000.0, + t_grep_regex.as_secs_f64() * 1_000_000.0 / n_grep_regex.max(1) as f64 + ); + eprintln!( + "║ Fuzzy find: {:>4} calls {:>8.1}ms ({:>6.1}µs/call) ║", + n_fuzzy, + t_fuzzy.as_secs_f64() * 1000.0, + t_fuzzy.as_secs_f64() * 1_000_000.0 / n_fuzzy.max(1) as f64 + ); + eprintln!( + "║ Dead checks: {:>4} calls {:>8.1}ms ({:>6.1}µs/call) ║", + n_dead, + t_dead_check.as_secs_f64() * 1000.0, + t_dead_check.as_secs_f64() * 1_000_000.0 / n_dead.max(1) as f64 + ); + eprintln!("║ ── Other ────────────────────────────────────────── ║"); + eprintln!( + "║ Mutations + FS I/O: {:>8.1}ms ║", + t_mutations.as_secs_f64() * 1000.0 + ); + eprintln!("╚══════════════════════════════════════════════════════╝"); +} + +fn write_diverse_file(dir: &Path, name: &str, token: &str, index: usize) { + let domain_idx = index % DOMAINS.len(); + write_diverse_file_with_domain(dir, name, token, domain_idx); +} + +fn write_diverse_file_with_domain(dir: &Path, name: &str, token: &str, domain_idx: usize) { + let domain = DOMAINS[domain_idx % DOMAINS.len()]; + let content = format!( + "// File: {name}\n\ + // Domain content for bigram diversity\n\ + {domain}\n\ + // === Unique searchable token below ===\n\ + const MARKER: &str = \"{token}\";\n\ + fn marker_function_{token}() {{ println!(\"{token}\"); }}\n" + ); + + if let Some(parent) = PathBuf::from(name).parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(dir.join(parent)).unwrap(); + } + } + fs::write(dir.join(name), content).unwrap(); +} + +// ═══════════════════════════════════════════════════════════════════════ +// Search helpers +// ═══════════════════════════════════════════════════════════════════════ + +fn grep_plain_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +fn grep_regex_opts() -> GrepSearchOptions { + GrepSearchOptions { + mode: GrepMode::Regex, + ..grep_plain_opts() + } +} + +fn grep_plain_count(picker: &FilePicker, query: &str) -> usize { + let parsed = parse_grep_query(query); + picker.grep(&parsed, &grep_plain_opts()).matches.len() +} + +fn grep_regex_count(picker: &FilePicker, regex_query: &str) -> usize { + let parsed = parse_grep_query(regex_query); + picker.grep(&parsed, &grep_regex_opts()).matches.len() +} + +/// Build a regex pattern from a token: `{first5}.*{last5}`. +/// For tokens shorter than 10 chars, just use the literal (escaped). +fn build_regex_pattern(token: &str) -> String { + if token.len() >= 10 { + let first5 = &token[..5]; + let last5 = &token[token.len() - 5..]; + format!("{}.*{}", regex_escape(first5), regex_escape(last5)) + } else { + regex_escape(token) + } +} + +/// Escape regex metacharacters in a string. +fn regex_escape(s: &str) -> String { + let mut escaped = String::with_capacity(s.len() + 4); + for ch in s.chars() { + match ch { + '.' | '*' | '+' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\\' | '^' | '$' | '|' => { + escaped.push('\\'); + escaped.push(ch); + } + _ => escaped.push(ch), + } + } + escaped +} + +/// Extract a fuzzy-searchable stem from a filename. +/// Strips the extension and any leading path components, keeping the bare name. +fn extract_stem(name: &str) -> String { + let p = PathBuf::from(name); + p.file_stem() + .unwrap_or_default() + .to_string_lossy() + .to_string() +} + +fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +fn wait_for_bigram(shared_picker: &SharedFilePicker) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(50)); + let ready = shared_picker + .read() + .ok() + .map(|guard| { + guard + .as_ref() + .map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } +} + +fn git_run(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .output() + .unwrap_or_else(|e| panic!("git {:?} failed: {}", args, e)); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); +} + +fn epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +fn git_init_and_commit(dir: &Path) { + git_run(dir, &["init"]); + git_run(dir, &["add", "-A"]); + git_run(dir, &["commit", "-m", "initial"]); +} + +/// Proves that dropping the picker while post-scan (warmup + bigram build) +/// is actively iterating raw pointers does NOT segfault. The Drop impl +/// sets `cancelled`, waits for `post_scan_indexing_active` to clear, and +/// only then frees the backing Vec. +/// +/// Runs 10 iterations to exercise the race window reliably. +#[test] +fn drop_during_post_scan_does_not_crash() { + let mut caught_active = 0u32; + + for round in 0..10 { + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // Create enough files so bigram build takes measurable time + for i in 0..2000 { + let dir = base.join(format!("d_{:02}", i % 20)); + fs::create_dir_all(&dir).unwrap(); + let content = format!( + "fn func_{i}() {{ let x = {i}; println!(\"{{x}}\"); }}\n\ + const T_{i}: &str = \"TOKEN_{i}\";\n" + ); + fs::write(dir.join(format!("f_{i:04}.rs")), content).unwrap(); + } + + git_init_and_commit(base); + + let shared_picker = SharedFilePicker::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + SharedFrecency::noop(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + watch: false, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .unwrap(); + + // Wait for scan but NOT for bigram — drop while post-scan is active + shared_picker.wait_for_scan(Duration::from_secs(10)); + + // Poll until post_scan_indexing_active is true (bigram started) + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let mut was_active = false; + loop { + if let Ok(guard) = shared_picker.read() { + if let Some(picker) = guard.as_ref() { + if picker.is_post_scan_active() { + was_active = true; + break; + } + } + } + if std::time::Instant::now() > deadline { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + + if was_active { + caught_active += 1; + } + + // Drop the picker while post_scan_indexing_active is set. + // Take it out of the shared handle first, then drop outside the lock — + // Drop spins until post-scan finishes, which needs the write lock for + // bigram install, so we can't hold it during Drop. + let old_picker = shared_picker.write().unwrap().take(); + drop(old_picker); // Drop fires here — spins until post-scan exits + + assert!( + shared_picker.read().unwrap().is_none(), + "round {round}: picker should be None after drop" + ); + } + + // The primary invariant — dropping while post-scan may be active must not + // crash — is exercised every round regardless. Catching the active window + // is timing-dependent: with a fast walker/scan the post-scan phase can + // complete before the poll observes it, especially on loaded CI runners. + // So we only warn (not fail) if no round observed it. + if caught_active == 0 { + eprintln!( + "warning: never observed post_scan_indexing_active=true; \ + drop-safety was still exercised in all rounds ({caught_active}/10)" + ); + } + eprintln!("Caught post-scan active in {caught_active}/10 rounds"); +} diff --git a/crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions b/crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions new file mode 100644 index 0000000..b092012 --- /dev/null +++ b/crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 2c9d1ea2efbf6161f84b69598e884dbf1bde6039c70625adde0374817e20e2ea +cc 1ac0f8f02b160dce13ca3f3630266abd24bd32b4e36d72a6a0a5365139ded3a8 diff --git a/crates/fff-core/tests/fuzz_git_watcher_stress.rs b/crates/fff-core/tests/fuzz_git_watcher_stress.rs new file mode 100644 index 0000000..dda7368 --- /dev/null +++ b/crates/fff-core/tests/fuzz_git_watcher_stress.rs @@ -0,0 +1,1671 @@ +//! Vibe coded stress test: randomized file + git operations driven against the *real* +//! `BackgroundWatcher`, asserting the git-status invariant after every mutation. +//! +//! ## What this test is trying to catch +//! +//! The background watcher is supposed to keep every indexed file's +//! `git_status` in sync with the actual repository state at all times. +//! There are two code paths that can drift: +//! +//! 1. **Per-file updates** — when regular files are created / modified the +//! watcher queries `git_status_for_paths(&[changed_file])`. If the git +//! state changes for files that weren't in the batch (rename detection, +//! submodule paths, index modifications applied by a sibling process), +//! those other files keep a stale `git_status` until the next full +//! rescan is triggered. +//! +//! 2. **Full rescans** — triggered by changes under `.git/` (index, HEAD, +//! MERGE_HEAD, etc.) and by `.gitignore` edits. A missed event here is +//! the most common failure mode: the watcher coalesced or dropped the +//! `.git/index` notification and the picker never learns that e.g. a +//! `git commit` cleared every `WT_MODIFIED`. +//! +//! ## How the test works +//! +//! * Spin up a real `FilePicker` with `watch: true` on a fresh temp repo. +//! * Use `proptest` to generate a sequence of 20–40 randomized ops (heavy on +//! git mutations: add / commit / reset / stash / gitignore edits). +//! * After **every** op, poll until the picker's per-file `git_status` agrees +//! with `git2::Repository::statuses()` verbatim, or bail with a rich diff. +//! * If there is ever a divergence that doesn't resolve within +//! `CONVERGE_TIMEOUT`, the test fails with the list of `(path, truth, +//! picker)` mismatches. +//! +//! The shape of the scenario is shrinkable: on failure proptest will shrink +//! the `Vec` so the reported diff corresponds to the smallest +//! surviving sequence. +//! +//! ## Runtime +//! +//! Each scenario does a real filesystem scan + watcher setup (~0.5–1 s) and +//! then runs 20–40 ops each of which waits for event propagation +//! (~100–500 ms on macOS FSEvents). With `cases = 2` the test takes +//! ~30–45 s on a typical dev machine — too slow to run as part of the +//! default `cargo test`, so it's gated behind the `stress` cfg. +//! +//! Run it explicitly with: +//! ```sh +//! RUSTFLAGS="--cfg stress" cargo test -p fff-search --test fuzz_git_watcher_stress -- --nocapture +//! ``` +//! +//! Or increase coverage via env: +//! ```sh +//! FFF_STRESS_CASES=8 FFF_STRESS_MAX_OPS=60 \ +//! RUSTFLAGS="--cfg stress" cargo test -p fff-search --test fuzz_git_watcher_stress -- --nocapture +//! ``` + +#![cfg(stress)] +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{ + FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser, SharedFilePicker, + SharedFrecency, +}; +use git2::{Repository, Status, StatusOptions}; +use proptest::prelude::*; +use proptest::strategy::ValueTree; +use proptest::test_runner::{ + Config as ProptestConfig, FileFailurePersistence, RngAlgorithm, TestRng, TestRunner, +}; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +/// Upper bound on how long we wait for the picker to converge after any op. +/// The watcher debounce is 50 ms; worst-case FSEvents propagation + queued +/// full-rescan can easily reach several seconds. 15 s is very generous. +const CONVERGE_TIMEOUT: Duration = Duration::from_secs(15); + +/// Poll interval while waiting for convergence. +const CONVERGE_POLL: Duration = Duration::from_millis(50); + +/// Small pause between back-to-back ops to simulate real user behavior +const PER_OP_SETTLE: Duration = Duration::from_millis(10); + +fn stress_cases() -> u32 { + std::env::var("FFF_STRESS_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2) +} + +fn stress_max_ops() -> usize { + std::env::var("FFF_STRESS_MAX_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(40) +} + +fn stress_min_ops() -> usize { + std::env::var("FFF_STRESS_MIN_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(20) +} + +/// A single randomized action. The runtime interprets abstract handles +/// (`idx`) by modulo against the *current* list of live files, so the same +/// abstract op sequence is always runnable regardless of which files exist. +#[derive(Debug, Clone)] +enum AbstractOp { + CreateFile { + seed: u32, + content_seed: u32, + }, + EditFile { + idx: usize, + content_seed: u32, + }, + DeleteFile { + idx: usize, + }, + RenameFile { + idx: usize, + new_seed: u32, + }, + CreateSubdirFile { + dir_seed: u16, + file_seed: u16, + content_seed: u32, + }, + GitignoreAppend { + pattern_seed: u16, + }, + GitAddAll, + GitCommit { + msg_seed: u16, + }, + GitResetHard, + GitStashThenPop, + Touch { + idx: usize, + }, // rewrite with same content; should still fire Modify + Noop, +} + +fn op_strategy() -> impl Strategy { + // Weights tuned to mirror a real editing session rather than a + // contrived git-heavy workload. In descending order: + // + // * edits dominate (~60%) — a real user bangs on Save all day + // * creates are moderate (~15%) — new files appear occasionally + // * deletes and renames are rare (~10%) — destructive ops happen + // but far less often than edits + // * explicit git operations are the rarest (~8%) — users stage / + // commit / reset in bursts between long stretches of editing + // + // The file-mutation path still exercises the per-path git-status + // update on every op via the watcher, so the git-status updater + // is *always* under load — the rare explicit `GitAddAll` / + // `GitCommit` / `GitResetHard` ops layer in coverage of the + // `.git/index` event → full-rescan path specifically. A 40-op + // scenario fires that path ~3 times, which is enough to keep the + // race between worktree writes and `.git/*` updates in rotation. + prop_oneof![ + 40 => (any::(), any::()).prop_map(|(i, c)| AbstractOp::EditFile { + idx: i, content_seed: c, + }), + 5 => any::().prop_map(|i| AbstractOp::Touch { idx: i }), + 8 => (any::(), any::()).prop_map(|(a, b)| AbstractOp::CreateFile { + seed: a, content_seed: b, + }), + 3 => (any::(), any::(), any::()).prop_map( + |(d, f, c)| AbstractOp::CreateSubdirFile { + dir_seed: d, file_seed: f, content_seed: c, + } + ), + 4 => any::().prop_map(|i| AbstractOp::DeleteFile { idx: i }), + 3 => (any::(), any::()).prop_map(|(i, s)| AbstractOp::RenameFile { + idx: i, new_seed: s, + }), + 1 => Just(AbstractOp::GitAddAll), + 1 => any::().prop_map(|m| AbstractOp::GitCommit { msg_seed: m }), + 1 => Just(AbstractOp::GitStashThenPop), + 1 => Just(AbstractOp::GitResetHard), + 1 => any::().prop_map(|p| AbstractOp::GitignoreAppend { pattern_seed: p }), + 2 => Just(AbstractOp::Noop), + ] +} + +fn ops_strategy() -> impl Strategy> { + ops_strategy_bounded(stress_min_ops(), stress_max_ops()) +} + +fn ops_strategy_bounded(min: usize, max: usize) -> impl Strategy> { + prop::collection::vec(op_strategy(), min..=max) +} + +const DEFAULT_STRESS_SEED: u64 = 0xDEAD_BEEF_CAFE_BABE; + +fn proptest_config() -> ProptestConfig { + ProptestConfig { + cases: stress_cases(), + // Cap shrinking because each shrink iteration replays the whole + // scenario (seconds of real IO). Proptest's default 4096 is wild. + max_shrink_iters: 16, + // `fork: true` would isolate runs but swallows the panic payload + // and printed diff through `rusty-fork`'s wire format — we want + // the convergence report visible on stderr, so we keep in-process. + // The watcher is fully torn down between cases via `Drop`. + fork: false, + // Integration tests don't live alongside a `lib.rs`/`main.rs`, so + // proptest's default `SourceParallel` persistence strategy can't + // locate the source tree and logs a noisy warning on every run. + // Pin the regression file to an explicit path next to this test + // so failing seeds get checked in and reproduced in CI. + failure_persistence: Some(Box::new(FileFailurePersistence::Direct(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fuzz_git_watcher_stress.proptest-regressions", + )))), + ..ProptestConfig::default() + } +} + +proptest! { + #![proptest_config(proptest_config())] + + /// Random-seeded scenario. Proptest picks a fresh seed every run from + /// system entropy, so CI sees different trajectories on every build + /// while known-bad seeds remain pinned in the regressions file. + #[test] + fn stress_random(ops in ops_strategy()) { + run_stress_scenario(&ops); + } +} + +/// Deterministic scenario keyed off `FFF_STRESS_SEED` (or +/// [`DEFAULT_STRESS_SEED`] if the env var is not set). Runs the same +/// proptest `ops_strategy()` but uses a ChaCha RNG seeded by the expanded +/// u64, so the exact case sequence is reproducible across machines and CI +/// runs. +/// +/// When this test panics, the panic message includes the seed so you can +/// re-run the failing case locally via: +/// +/// ```sh +/// RUSTFLAGS="--cfg stress" FFF_STRESS_SEED=0xDEADBEEFCAFEBABE \ +/// cargo test -p fff-search --test fuzz_git_watcher_stress seeded -- --nocapture +/// ``` +#[test] +fn stress_seeded() { + let seed = parse_stress_seed(); + let seed_bytes = expand_u64_seed(seed); + + eprintln!("stress_seeded: using deterministic seed {seed:#018x}"); + + let mut config = proptest_config(); + // The seeded run should never write to the shared regressions file — + // its failures are reproducible from the env var alone, and polluting + // the shared file with the deterministic seed would mask regressions + // for the random run that actually needs persistence. + config.failure_persistence = Some(Box::new(FileFailurePersistence::Off)); + + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &seed_bytes); + let mut runner = TestRunner::new_with_rng(config, rng); + let strategy = ops_strategy(); + + // Mimic `proptest!`'s case loop: run `Config::cases` independent draws + // from the strategy, failing the test as soon as any one scenario + // diverges. We drive this ourselves because `runner.run()` can't + // accept a non-Fn closure, and our scenario runner is side-effectful. + for case_idx in 0..runner.config().cases { + let tree = strategy + .new_tree(&mut runner) + .expect("ops_strategy::new_tree"); + let ops = tree.current(); + eprintln!( + " case {}/{}: {} ops", + case_idx + 1, + runner.config().cases, + ops.len() + ); + run_stress_scenario(&ops); + } +} + +/// Pinned deterministic regression for the git-status divergence found on +/// Windows CI (run 28264744320): after a `GitCommit` the picker retained stale +/// `INDEX_*` bits because a pre-commit per-path status snapshot was applied +/// after the post-commit full rescan. +/// +/// The op sequence is regenerated from the proptest seed persisted in the +/// regressions file (`cc 2c9d...`) using the CI op bounds (30..=60) that were +/// in effect when the failure was found. The fingerprint assertion fails +/// loudly if `ops_strategy()` ever changes shape — a changed strategy would +/// silently decode the same seed into a *different* scenario, turning this +/// regression guard into a no-op. +#[test] +fn stress_regression_stale_index_after_commit() { + let ops = ops_from_chacha_seed(REGRESSION_SEED_HEX, 30, 60); + assert_eq!( + (ops.len(), fingerprint_ops(&ops)), + (59, 0xc73f_16ce_b249_78eb), + "ops_strategy() changed shape: the pinned seed no longer decodes to \ + the original Windows-CI scenario. Either revert the strategy change \ + or re-pin this regression (the original literal op list is in git \ + history of this file).", + ); + run_stress_scenario(&ops); +} + +/// 32-byte ChaCha seed persisted by proptest for the Windows CI failure +/// (the `cc 2c9d...` entry in the regressions file). +const REGRESSION_SEED_HEX: &str = + "2c9d1ea2efbf6161f84b69598e884dbf1bde6039c70625adde0374817e20e2ea"; + +/// Regenerate an op sequence from a persisted proptest ChaCha seed by +/// replaying `ops_strategy()` the same way proptest does for regressions. +/// `min`/`max` must match the `FFF_STRESS_{MIN,MAX}_OPS` bounds that were +/// in effect when the seed was persisted — the strategy's value tree +/// depends on them. +fn ops_from_chacha_seed(seed_hex: &str, min: usize, max: usize) -> Vec { + let seed_bytes: Vec = (0..seed_hex.len() / 2) + .map(|i| u8::from_str_radix(&seed_hex[2 * i..2 * i + 2], 16).expect("valid hex seed")) + .collect(); + let mut config = proptest_config(); + config.failure_persistence = Some(Box::new(FileFailurePersistence::Off)); + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &seed_bytes); + let mut runner = TestRunner::new_with_rng(config, rng); + ops_strategy_bounded(min, max) + .new_tree(&mut runner) + .expect("ops_strategy::new_tree") + .current() +} + +/// FNV-1a over the debug repr of the ops; stable across platforms and runs. +fn fingerprint_ops(ops: &[AbstractOp]) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in format!("{ops:?}").bytes() { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// Parse `FFF_STRESS_SEED` as either decimal or `0x`-prefixed hex. +fn parse_stress_seed() -> u64 { + match std::env::var("FFF_STRESS_SEED") { + Ok(raw) => { + let trimmed = raw.trim(); + if let Some(hex) = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + { + u64::from_str_radix(hex, 16) + .unwrap_or_else(|e| panic!("FFF_STRESS_SEED={raw:?} is not valid hex: {e}")) + } else { + trimmed + .parse::() + .unwrap_or_else(|e| panic!("FFF_STRESS_SEED={raw:?} is not a valid u64: {e}")) + } + } + Err(_) => DEFAULT_STRESS_SEED, + } +} + +/// Expand a u64 into the 32-byte seed ChaCha20 wants, by repeating the +/// little-endian bytes four times. The cycle is deliberate: two different +/// u64 seeds produce completely different byte sequences, so collisions +/// across the expansion are irrelevant in practice. +fn expand_u64_seed(seed: u64) -> [u8; 32] { + let le = seed.to_le_bytes(); + let mut out = [0u8; 32]; + for i in 0..4 { + out[i * 8..(i + 1) * 8].copy_from_slice(&le); + } + out +} + +#[derive(Debug)] +struct Live { + /// Path relative to the repo root (forward slashes on all platforms). + relative: String, + abs: PathBuf, +} + +fn run_stress_scenario(ops: &[AbstractOp]) { + // Opt-in tracing so `RUST_LOG=fff_search=debug` shows the watcher / + // refresh / update trail when debugging a failing run. Uses + // `try_init` so proptest can run many scenarios in the same process + // without double-initialising the subscriber. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + // Default to DEBUG for fff crates so CI failures on + // flaky OSes (Windows) include the watcher/event trail + // without needing to re-run with RUST_LOG set. + .unwrap_or_else(|_| { + tracing_subscriber::EnvFilter::new( + "warn,fff_search=debug,notify=debug,notify_debouncer_full=debug", + ) + }), + ) + .with_test_writer() + .try_init(); + + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + seed_repo(&base); + + let (shared_picker, _frecency) = start_watched_picker(&base); + wait_ready(&shared_picker); + + // Reconcile `live` from disk: after `git reset --hard` or similar the + // worktree can change under us, so we resync at every step instead of + // trusting our in-memory list. + let mut live: Vec = get_baseline_status_from_git(&base); + + // Sleep past the coarse (1 s) mtime tick so subsequent edits definitely + // advance mtime. Not strictly required for git status correctness but + // matches what real users experience. + std::thread::sleep(Duration::from_millis(1100)); + + for (step, op) in ops.iter().enumerate() { + apply_op(op, &base, &mut live); + + // Re-sync live from disk — ops like GitResetHard / GitStashThenPop + // may have created or removed files behind our back. + live = get_baseline_status_from_git(&base); + std::thread::sleep(PER_OP_SETTLE); + + if let Err(err) = converge_git_status(&shared_picker, &base, &live) { + panic!( + "\n──────────────────────────────────────────────────────────\n\ + ❌ Picker git_status diverged from repository truth.\n\ + ──────────────────────────────────────────────────────────\n\ + step: {step}\n\ + op: {op:?}\n\ + scenario ops:\n{trace}\n\ + ──────────────────────────────────────────────────────────\n\ + {err}\n", + trace = format_ops_trace(ops, step), + ); + } + } +} + +fn format_ops_trace(ops: &[AbstractOp], up_to_and_including: usize) -> String { + let mut s = String::new(); + for (i, op) in ops.iter().enumerate().take(up_to_and_including + 1) { + s.push_str(&format!(" [{i:>3}] {op:?}\n")); + } + s +} + +fn seed_repo(base: &Path) { + fs::create_dir_all(base.join("src")).unwrap(); + fs::write(base.join("README.md"), "# seed\n").unwrap(); + fs::write(base.join("src/main.rs"), "fn main() {}\n").unwrap(); + fs::write(base.join("src/lib.rs"), "// lib\n").unwrap(); + fs::write(base.join(".gitignore"), "*.log\ntmp/\n").unwrap(); + + git(base, &["init", "-b", "main"]); + git(base, &["config", "user.email", "fuzz@fff.test"]); + git(base, &["config", "user.name", "fuzz"]); + // Disable rename detection noise — libgit2 still computes ranks, but + // keeping the porcelain behaviour deterministic helps with diff reading. + git(base, &["config", "status.renames", "false"]); + git(base, &["add", "-A"]); + git(base, &["commit", "-m", "seed", "--no-gpg-sign"]); +} + +fn apply_op(op: &AbstractOp, base: &Path, live: &mut [Live]) { + use AbstractOp::*; + + match op { + CreateFile { seed, content_seed } => { + let rel = format!("f_{seed:08x}.rs"); + let abs = base.join(&rel); + if abs.exists() { + return; + } + fs::write(&abs, content_for(*content_seed, &rel)).unwrap(); + } + EditFile { idx, content_seed } => { + if live.is_empty() { + return; + } + let i = idx % live.len(); + let abs = &live[i].abs; + if !abs.is_file() { + return; + } + // Body must differ so git sees WT_MODIFIED (not just atime touch). + let body = format!( + "// edited seed={content_seed:08x}\n{}", + content_for(*content_seed, &live[i].relative) + ); + fs::write(abs, body).unwrap(); + } + Touch { idx } => { + // Rewrite with the same bytes we currently hold on disk. Still + // generates a Modify event; git status stays the same if the + // content matches the index, or WT_MODIFIED if it differs. + if live.is_empty() { + return; + } + let i = idx % live.len(); + let abs = &live[i].abs; + if let Ok(contents) = fs::read(abs) { + let _ = fs::write(abs, contents); + } + } + DeleteFile { idx } => { + if live.is_empty() { + return; + } + let i = idx % live.len(); + let _ = fs::remove_file(&live[i].abs); + } + RenameFile { idx, new_seed } => { + if live.is_empty() { + return; + } + let i = idx % live.len(); + let old = &live[i].abs; + if !old.is_file() { + return; + } + let new_rel = format!("r_{new_seed:08x}.rs"); + let new_abs = base.join(&new_rel); + if new_abs.exists() { + return; + } + let _ = fs::rename(old, &new_abs); + } + CreateSubdirFile { + dir_seed, + file_seed, + content_seed, + } => { + let rel = format!("d_{dir_seed:04x}/inside_{file_seed:04x}.rs"); + let abs = base.join(&rel); + if abs.exists() { + return; + } + fs::create_dir_all(abs.parent().unwrap()).unwrap(); + fs::write(&abs, content_for(*content_seed, &rel)).unwrap(); + } + GitignoreAppend { pattern_seed } => { + // Introduce NEW ignore patterns for a namespace not used by any + // other op so we never accidentally re-ignore a file under test. + let pattern = format!("__ignored_{pattern_seed:x}/\n"); + let gi = base.join(".gitignore"); + let mut cur = fs::read_to_string(&gi).unwrap_or_default(); + cur.push_str(&pattern); + fs::write(&gi, cur).unwrap(); + } + GitAddAll => { + git_allow_fail(base, &["add", "-A"]); + } + GitCommit { msg_seed } => { + // `--allow-empty` so we don't depend on there actually being + // staged changes; this still bumps HEAD and rewrites .git/index. + let _ = git_output( + base, + &[ + "commit", + "-m", + &format!("fuzz-{msg_seed:x}"), + "--allow-empty", + "--allow-empty-message", + "--no-gpg-sign", + ], + ); + } + GitResetHard => { + git_allow_fail(base, &["reset", "--hard"]); + } + GitStashThenPop => { + git_allow_fail(base, &["add", "-A"]); + let stash = git_output(base, &["stash", "push", "-u", "-m", "fuzz"]); + let had_stash = stash + .as_ref() + .map(|o| { + o.status.success() + && !String::from_utf8_lossy(&o.stdout).contains("No local changes") + }) + .unwrap_or(false); + if had_stash { + git_allow_fail(base, &["stash", "pop"]); + } + } + Noop => {} + } +} + +/// Block until the picker's git status agrees with `git2::Repository::statuses`. +/// +/// Poll until the picker's git-status view matches libgit2's truth AND +/// the real-query probe succeeds, or bail with a rich diff. +/// +/// On timeout, returns an `Err` describing every disagreement. +fn converge_git_status( + shared_picker: &SharedFilePicker, + base: &Path, + live: &[Live], +) -> Result<(), String> { + let deadline = Instant::now() + CONVERGE_TIMEOUT; + let mut last_mismatches: Vec; + let mut last_probe_err: Option = None; + + loop { + let truth = read_truth_status(base); + let picker_view = read_picker_status(shared_picker); + last_mismatches = diff_statuses(&truth, &picker_view); + + // Real-query probe: one random live file per round via fuzzy + grep. + // We only consider the probe authoritative once the main git-status + // enumeration agrees — otherwise a probe failure might just be + // the same debouncer-lag we're already waiting out. + let probe = if last_mismatches.is_empty() { + probe_real_queries(shared_picker, live) + } else { + None + }; + + match (last_mismatches.is_empty(), probe) { + (true, None) | (true, Some(Ok(()))) => return Ok(()), + (true, Some(Err(msg))) => last_probe_err = Some(msg), + _ => {} + } + + if Instant::now() >= deadline { + let mut report = if !last_mismatches.is_empty() { + format_mismatches(&last_mismatches, shared_picker) + } else { + String::from("git_status enumeration converged, but real-query probe failed:\n") + }; + if let Some(probe_msg) = last_probe_err { + report.push_str("\n── real-query probe ──\n"); + report.push_str(&probe_msg); + report.push('\n'); + } + report.push_str(&debug_dump_environment( + base, + shared_picker, + &last_mismatches, + )); + return Err(report); + } + std::thread::sleep(CONVERGE_POLL); + } +} + +/// On-failure diagnostic dump. Includes: +/// * base path (raw + OS encoding) so we can spot UNC/`\\?\` prefixes on Windows +/// * picker's own `base_path()` for comparison +/// * direct libgit2 `status_file` probes for every mismatched path +/// * full picker enumeration (first 20 rows) so we can see what keys +/// the picker is returning vs. what git2 reports +fn debug_dump_environment( + base: &Path, + shared_picker: &SharedFilePicker, + mismatches: &[Mismatch], +) -> String { + let mut s = String::new(); + s.push_str("\n── diagnostic dump ──\n"); + s.push_str(&format!( + "test base path (display) : {}\n", + base.display() + )); + s.push_str(&format!("test base path (debug) : {:?}\n", base)); + s.push_str(&format!( + "test base path (os bytes) : {:?}\n", + base.as_os_str() + )); + + if let Ok(guard) = shared_picker.read() + && let Some(picker) = guard.as_ref() + { + s.push_str(&format!( + "picker base_path (display) : {}\n", + picker.base_path().display() + )); + s.push_str(&format!( + "picker base_path (debug) : {:?}\n", + picker.base_path() + )); + } + + // Probe libgit2 directly for each mismatched path. + if let Ok(repo) = Repository::open(base) { + s.push_str("\nlibgit2 status_file probes (per mismatch path):\n"); + for m in mismatches { + let path = match m { + Mismatch::Disagree { path, .. } => path, + Mismatch::ExtraInPicker { path, .. } => path, + }; + match repo.status_file(std::path::Path::new(path)) { + Ok(st) => s.push_str(&format!(" • {path} -> {st:?}\n")), + Err(e) => { + s.push_str(&format!( + " • {path} -> ERROR {} (class={:?}, code={:?})\n", + e.message(), + e.class(), + e.code() + )); + } + } + } + } else { + s.push_str("\nlibgit2: could not open repo at base path\n"); + } + + // Dump the raw picker enumeration for comparison — up to 20 rows. + s.push_str("\npicker enumeration (first 20 entries, with byte-repr of each relative path):\n"); + if let Ok(guard) = shared_picker.read() + && let Some(picker) = guard.as_ref() + { + let parser = QueryParser::default(); + let parsed = parser.parse(""); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + ..Default::default() + }, + ); + for (i, f) in result.items.iter().take(20).enumerate() { + let raw = f.relative_path(picker); + let norm = normalize(raw.clone()); + s.push_str(&format!( + " [{i:>2}] raw={raw:?} norm={norm:?} status={:?}\n", + f.git_status + )); + } + s.push_str(&format!(" (total: {} items)\n", result.items.len())); + } + + s +} + +#[derive(Debug)] +enum Mismatch { + /// Git knows about this path with `truth`, picker has `picker` (or None = missing). + Disagree { + path: String, + truth: Status, + picker: Option>, + }, + /// Picker has a non-clean entry for a path git doesn't report at all. + ExtraInPicker { path: String, picker: Status }, +} + +fn read_truth_status(base: &Path) -> BTreeMap { + let repo = Repository::open(base).expect("open repo for truth"); + let mut opts = StatusOptions::new(); + opts.include_untracked(true) + .recurse_untracked_dirs(true) + .include_unmodified(true) + .exclude_submodules(true); + let statuses = repo.statuses(Some(&mut opts)).expect("read statuses"); + + let mut out = BTreeMap::new(); + for entry in statuses.iter() { + if let Some(p) = entry.path() { + // git2 returns forward-slash paths; accept as-is. + out.insert(p.to_string(), entry.status()); + } + } + out +} + +fn read_picker_status(shared: &SharedFilePicker) -> BTreeMap> { + let guard = shared.read().expect("picker read lock"); + let picker = guard.as_ref().expect("picker initialized"); + + // Use the user-facing `fuzzy_search` API with an empty query to enumerate + // every file a user could ever see in the picker. An empty fuzzy query + // falls through to the frecency-only scoring path which returns ALL + // non-deleted files (base + overflow). A very large `limit` makes sure + // we don't silently paginate anything away for scenarios with many files. + // + // This intentionally mirrors what a real Neovim user observes — soft- + // deleted tombstones, files filtered out by search-side predicates, etc. + // are all excluded from the picker's user-facing view. + let parser = QueryParser::default(); + let parsed = parser.parse(""); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 100_000, + }, + ..Default::default() + }, + ); + + let mut out = BTreeMap::new(); + for f in &result.items { + out.insert(normalize(f.relative_path(picker)), f.git_status); + } + out +} + +fn normalize(s: String) -> String { + #[cfg(windows)] + { + if s.contains('\\') { + return s.replace('\\', "/"); + } + } + s +} + +/// Probe a single file by name via the public `fuzzy_search` API and return +/// its status, or `None` if the file is not findable. Used for diagnostics +/// when the top-level enumeration reports a disagreement — confirms that +/// the mismatch reproduces via the exact user-facing lookup path. +fn probe_single_file_status(shared: &SharedFilePicker, relative: &str) -> Option> { + let guard = shared.read().ok()?; + let picker = guard.as_ref()?; + let parser = QueryParser::default(); + let parsed = parser.parse(relative); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .find(|f| normalize(f.relative_path(picker)) == relative) + .map(|f| f.git_status) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Real-query probes +// ═══════════════════════════════════════════════════════════════════════════ +// +// These exercise the **user-facing** fuzzy + grep surfaces with queries that +// look like what a human actually types, on top of a live file picked at +// random from the on-disk truth. They run once per convergence round in +// addition to the primary git-status enumeration check, and add coverage +// for three paths the empty-query enumeration never touches: +// +// * fuzzy matching with a non-empty query (bigram prefilter + score) +// * path-constraint fuzzy queries ("foo src/") +// * live grep (mmap content cache + bigram overlay for overflow files) +// +// An atomic counter drives the rotation across rounds so the probe spreads +// its attention across every live file over the course of a scenario. +static PROBE_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Read a file on disk and pull out the `FFF_STRESS_MARKER_` token +/// that [`content_for`] embedded in it. Returns `None` for files we never +/// wrote (seed README.md, .gitignore) — the caller skips the grep probe +/// in that case but still runs the fuzzy probe. +fn extract_marker(abs: &Path) -> Option { + let content = fs::read_to_string(abs).ok()?; + let start = content.find("FFF_STRESS_MARKER_")?; + // Marker is exactly `FFF_STRESS_MARKER_` + 8 hex chars. + const MARKER_LEN: usize = "FFF_STRESS_MARKER_".len() + 8; + if start + MARKER_LEN > content.len() { + return None; + } + let marker = &content[start..start + MARKER_LEN]; + // Sanity: the trailing 8 chars must all be hex. + if !marker.as_bytes()[MARKER_LEN - 8..] + .iter() + .all(|b| b.is_ascii_hexdigit()) + { + return None; + } + Some(marker.to_string()) +} + +/// File-stem → fuzzy search query. Strips extension + leading path +/// components so the probe feeds the picker a typical "I know roughly +/// what I'm looking for" query. +fn stem_for_query(relative: &str) -> String { + PathBuf::from(relative) + .file_stem() + .unwrap_or_default() + .to_string_lossy() + .into_owned() +} + +/// Run a one-shot fuzzy_search and return the matched items' relative +/// paths paired with their `git_status`. Uses only public APIs. +fn fuzzy_search_items(shared: &SharedFilePicker, query: &str) -> Vec<(String, Option)> { + let guard = match shared.read() { + Ok(g) => g, + Err(_) => return Vec::new(), + }; + let Some(picker) = guard.as_ref() else { + return Vec::new(); + }; + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 500, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| (normalize(f.relative_path(picker)), f.git_status)) + .collect() +} + +/// Run live grep (plain-text mode) and return the unique set of matched +/// file paths. A file appears in the set iff at least one line inside it +/// matches the query. Uses only public APIs. +fn grep_plain_matches(shared: &SharedFilePicker, query: &str) -> Vec { + let guard = match shared.read() { + Ok(g) => g, + Err(_) => return Vec::new(), + }; + let Some(picker) = guard.as_ref() else { + return Vec::new(); + }; + let parsed = parse_grep_query(query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let result = picker.grep(&parsed, &opts); + // `GrepResult::files` is the already-deduplicated list of files that + // contained at least one match — exactly what we want. + result + .files + .iter() + .map(|f| normalize(f.relative_path(picker))) + .collect() +} + +/// Run live grep (fuzzy mode) and return matched file paths. +/// Exercises the `fuzzy_grep_search` code path which resolves content +/// via arena pointers — the path that was silently broken for overflow +/// files before the overflow_arena fix. +fn grep_fuzzy_matches(shared: &SharedFilePicker, query: &str) -> Vec { + let guard = match shared.read() { + Ok(g) => g, + Err(_) => return Vec::new(), + }; + let Some(picker) = guard.as_ref() else { + return Vec::new(); + }; + let parsed = parse_grep_query(query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::Fuzzy, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let result = picker.grep(&parsed, &opts); + result + .files + .iter() + .map(|f| normalize(f.relative_path(picker))) + .collect() +} + +/// Run live grep (regex mode) and return matched file paths. +fn grep_regex_matches(shared: &SharedFilePicker, query: &str) -> Vec { + let guard = match shared.read() { + Ok(g) => g, + Err(_) => return Vec::new(), + }; + let Some(picker) = guard.as_ref() else { + return Vec::new(); + }; + let parsed = parse_grep_query(query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::Regex, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let result = picker.grep(&parsed, &opts); + result + .files + .iter() + .map(|f| normalize(f.relative_path(picker))) + .collect() +} + +/// Report from [`probe_real_queries`]. `None` means "nothing to probe this +/// round" (empty live set). `Some(Err)` means a probe disagreed with truth +/// — convergence should not treat this as success. +type ProbeOutcome = Option>; + +/// Real-query verification: for one rotated-live-file per round, run a +/// fuzzy search by stem and (when a marker is present) a grep for its +/// embedded token. Asserts both return the file *and* that its +/// `git_status` matches the picker's main view. +/// +/// Return values: +/// * `None` — nothing to probe (no live files). +/// * `Some(Ok(()))` — the probe agreed with truth on both surfaces. +/// * `Some(Err(s))` — diagnostic describing the disagreement. +fn probe_real_queries(shared: &SharedFilePicker, live: &[Live]) -> ProbeOutcome { + if live.is_empty() { + return None; + } + let idx = (PROBE_COUNTER.fetch_add(1, Ordering::Relaxed) as usize) % live.len(); + let target = &live[idx]; + + // --- Fuzzy probe: search by file stem --- + let stem = stem_for_query(&target.relative); + // Tiny stems (< 2 chars) are rejected by the fuzzy scorer and + // surface via the frecency fallback — skip the assertion in that + // case, nothing meaningful to verify. + if stem.len() >= 2 { + let fuzzy_hits = fuzzy_search_items(shared, &stem); + let found = fuzzy_hits.iter().find(|(p, _)| p == &target.relative); + if found.is_none() { + return Some(Err(format!( + "fuzzy_search({stem:?}) did not return expected live file {:?}\n\ + got {} results; first few: {:?}", + target.relative, + fuzzy_hits.len(), + fuzzy_hits.iter().take(5).collect::>(), + ))); + } + } + + // --- Grep probe: search for the content marker using a randomly + // rotated grep strategy. Each round picks one of PlainText / Fuzzy / + // Regex so over many rounds all three code paths get exercised, + // including the overflow-arena resolution that was previously broken + // in fuzzy grep. + if let Some(marker) = extract_marker(&target.abs) { + let probe_round = PROBE_COUNTER.load(Ordering::Relaxed); + let (mode_name, matches) = match probe_round % 3 { + 0 => ("plain", grep_plain_matches(shared, &marker)), + 1 => ("fuzzy", grep_fuzzy_matches(shared, &marker)), + _ => ("regex", grep_regex_matches(shared, &marker)), + }; + if !matches.contains(&target.relative) { + return Some(Err(format!( + "grep[{mode_name}]({marker:?}) did not return expected live file {:?}\n\ + got {} matched files; first few: {:?}", + target.relative, + matches.len(), + matches.iter().take(5).collect::>(), + ))); + } + } + + Some(Ok(())) +} + +/// Returns the full list of disagreements. Empty means "in sync". +/// +/// We deliberately exclude a few categories from being considered bugs: +/// * Paths that git marks as `WT_DELETED` / `INDEX_DELETED` but the picker +/// has already dropped — this is by design (deleted files leave the +/// index immediately). +/// * Paths that git marks as `IGNORED` but the picker has already dropped. +/// * Paths under `.git/` — never tracked by the picker. +fn diff_statuses( + truth: &BTreeMap, + picker: &BTreeMap>, +) -> Vec { + let mut out = Vec::new(); + + for (path, &truth_status) in truth { + if path.starts_with(".git/") || path == ".git" { + continue; + } + match picker.get(path) { + Some(&p) => { + if !status_equivalent(truth_status, p) { + out.push(Mismatch::Disagree { + path: path.clone(), + truth: truth_status, + picker: Some(p), + }); + } + } + None => { + // Tolerate picker not having deleted-from-disk files. + let only_absence_reasons = + Status::WT_DELETED | Status::INDEX_DELETED | Status::IGNORED; + if !truth_status.intersects(only_absence_reasons) { + out.push(Mismatch::Disagree { + path: path.clone(), + truth: truth_status, + picker: None, + }); + } + } + } + } + + for (path, &p) in picker { + if path.starts_with(".git/") || path == ".git" { + continue; + } + if !truth.contains_key(path) { + // Picker thinks a file exists that git has never heard of. + // Only a bug if the picker also thinks it has a non-clean + // status for it — otherwise it's a transient during which the + // picker has indexed a file before the next truth snapshot. + let non_clean = match p { + None => false, + Some(s) => !(s.is_empty() || s == Status::CURRENT), + }; + if non_clean { + out.push(Mismatch::ExtraInPicker { + path: path.clone(), + picker: p.unwrap_or(Status::CURRENT), + }); + } + } + } + + out +} + +/// `None` and `Some(CURRENT|empty)` are both "clean"; otherwise the bitsets +/// must match exactly. +fn status_equivalent(truth: Status, picker: Option) -> bool { + let picker_bits = picker.unwrap_or(Status::CURRENT); + let truth_clean = truth.is_empty() || truth == Status::CURRENT; + let picker_clean = picker_bits.is_empty() || picker_bits == Status::CURRENT; + if truth_clean && picker_clean { + return true; + } + truth == picker_bits +} + +fn format_mismatches(mismatches: &[Mismatch], shared: &SharedFilePicker) -> String { + let mut s = String::new(); + s.push_str(&format!( + "{} mismatch(es) after {} of wait:\n", + mismatches.len(), + humantime(CONVERGE_TIMEOUT), + )); + for m in mismatches { + match m { + Mismatch::Disagree { + path, + truth, + picker, + } => { + let probe = probe_single_file_status(shared, path); + s.push_str(&format!( + " • {path}\n truth : {}\n picker(enum): {}\n picker(probe): {}\n", + format_status(Some(*truth)), + match picker { + Some(p) => format_status(*p), + None => "".into(), + }, + match probe { + Some(p) => format_status(p), + None => "".into(), + } + )); + } + Mismatch::ExtraInPicker { path, picker } => { + let probe = probe_single_file_status(shared, path); + s.push_str(&format!( + " • {path}\n truth : \n picker(enum) : {}\n picker(probe): {}\n", + format_status(Some(*picker)), + match probe { + Some(p) => format_status(p), + None => "".into(), + } + )); + } + } + } + s +} + +fn format_status(s: Option) -> String { + match s { + None => "None (= clean)".into(), + Some(st) if st.is_empty() || st == Status::CURRENT => "CURRENT (= clean)".into(), + Some(st) => format!("{st:?}"), + } +} + +fn humantime(d: Duration) -> String { + format!("{:.1}s", d.as_secs_f64()) +} + +fn get_baseline_status_from_git(base: &Path) -> Vec { + let mut out = Vec::new(); + let repo = match Repository::open(base) { + Ok(r) => r, + Err(_) => return out, + }; + let mut opts = StatusOptions::new(); + opts.include_untracked(true) + .recurse_untracked_dirs(true) + .include_unmodified(true) + .exclude_submodules(true); + let statuses = match repo.statuses(Some(&mut opts)) { + Ok(s) => s, + Err(_) => return out, + }; + for entry in statuses.iter() { + if let Some(p) = entry.path() { + let abs = base.join(p); + // Must be a real file *right now* — ignore stale WT_DELETED rows. + if abs.is_file() { + out.push(Live { + relative: p.to_string(), + abs, + }); + } + } + } + out +} + +/// Content marker embedded in every generated file body. The probing +/// layer reads this back via `grep` to exercise the live-grep path +/// (which hits the bigram index / mmap cache / overflow content store — +/// code paths the empty-query fuzzy enumeration never touches). +/// +/// Format: `FFF_STRESS_MARKER_<32-bit hex>` — namespaced so a repo +/// search for "MARKER" finds nothing from this test outside files we +/// wrote ourselves, which keeps the grep assertion unambiguous. +fn marker_for(seed: u32) -> String { + format!("FFF_STRESS_MARKER_{seed:08x}") +} + +fn content_for(seed: u32, name: &str) -> String { + let marker = marker_for(seed); + // Keep it small but distinct so adjacent edits produce different SHAs. + // The marker appears twice — once in a comment, once in a string + // literal — so a grep for it always has at least two hits per file + // and one of them is guaranteed to be on its own line for easy + // assertion. + format!( + "// file: {name}\n\ + // seed: {seed:08x}\n\ + // anchor: {marker}\n\ + pub fn anchor_{seed:x}() {{ let _ = \"{marker}\"; }}\n" + ) +} + +fn start_watched_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::noop(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("FilePicker::new_with_shared_state"); + + (shared_picker, shared_frecency) +} + +fn wait_ready(p: &SharedFilePicker) { + assert!( + p.wait_for_scan(Duration::from_secs(15)), + "timed out waiting for initial scan" + ); + assert!( + p.wait_for_watcher(Duration::from_secs(15)), + "timed out waiting for watcher" + ); + // macOS FSEvents sometimes delivers a burst of "warmup" events right + // after the stream opens — let them drain before we start fuzzing. + std::thread::sleep(Duration::from_millis(200)); +} + +fn git_env() -> [(&'static str, &'static str); 4] { + [ + ("GIT_AUTHOR_NAME", "fuzz"), + ("GIT_AUTHOR_EMAIL", "fuzz@fff.test"), + ("GIT_COMMITTER_NAME", "fuzz"), + ("GIT_COMMITTER_EMAIL", "fuzz@fff.test"), + ] +} + +fn git(base: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(base) + .envs(git_env()) + .output() + .unwrap_or_else(|e| panic!("git {args:?}: {e}")); + assert!( + out.status.success(), + "git {args:?} failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git_allow_fail(base: &Path, args: &[&str]) { + let _ = Command::new("git") + .args(args) + .current_dir(base) + .envs(git_env()) + .output(); +} + +fn git_output(base: &Path, args: &[&str]) -> Option { + Command::new("git") + .args(args) + .current_dir(base) + .envs(git_env()) + .output() + .ok() +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Merge-conflict scenario +// ═══════════════════════════════════════════════════════════════════════════ +// +// The fuzz alphabet above intentionally avoids branch operations — merge +// conflicts need a very specific topology (two divergent edits to the same +// hunk) that random sampling would almost never hit. This scripted test +// fills that gap with a deterministic end-to-end conflict flow: +// +// 1. seed a single tracked file `conflict.rs` +// 2. branch to `feature`, rewrite the file's inner expression, commit +// 3. back on `main`, rewrite the same hunk differently, commit +// 4. `git merge feature` — leaves `.git/MERGE_HEAD` + conflict markers +// 5. picker must observe `Status::CONFLICTED` for `conflict.rs`, and the +// same real-query surfaces (fuzzy + grep for `<<<<<<<`) must return +// the conflicted file +// 6. resolve by writing the merged content, `git add`, `git commit` +// 7. picker must converge back to `Status::CURRENT` +// +// On top of the `--cfg stress` gate the test inherits (so it never runs +// under plain `cargo test`), this is cross-platform: it uses libgit2 +// internals + the `git` CLI, both of which work identically on macOS +// FSEvents, Linux inotify, and Windows ReadDirectoryChangesW (if we ever +// add Windows to the matrix). The convergence window is a bit larger than +// the fuzz case because `git merge` touches several `.git/*` files at +// once, producing a heftier event burst. + +/// Convergence timeout for the conflict flow. `git merge` fires more FS +/// events than a plain `git add` — `.git/ORIG_HEAD`, `.git/MERGE_HEAD`, +/// `.git/MERGE_MSG`, the worktree file itself — so the debouncer has +/// more work than a one-off refresh. 30 s is still very generous. +const CONFLICT_CONVERGE_TIMEOUT: Duration = Duration::from_secs(30); + +#[test] +fn stress_merge_conflict_convergence() { + // Opt-in tracing for parity with the other stress tests. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + // Default to DEBUG for fff crates so CI failures on + // flaky OSes (Windows) include the watcher/event trail + // without needing to re-run with RUST_LOG set. + .unwrap_or_else(|_| { + tracing_subscriber::EnvFilter::new( + "warn,fff_search=debug,notify=debug,notify_debouncer_full=debug", + ) + }), + ) + .with_test_writer() + .try_init(); + + let tmp = TempDir::new().expect("mktemp"); + let base = tmp.path().canonicalize().expect("canonicalize tmp"); + + seed_conflict_repo(&base); + + let (shared_picker, _frecency) = start_watched_picker(&base); + wait_ready(&shared_picker); + + // ───────────────────────────────────────────────────────────────── + // Stage 1: create divergent commits on `feature` and `main` + // ───────────────────────────────────────────────────────────────── + // + // `conflict.rs` on `main` contains `BASE` text at line 2. We rewrite + // that line two different ways on two branches, so a merge can't + // pick a side automatically. + // + // Each rewrite is preceded by a 1.1 s sleep so the file's mtime + // advances past the previous write's — the watcher's mmap cache + // invalidation is mtime-triggered at 1 s granularity, and without + // the sleep a follow-up grep on the picker sees stale content from + // whichever variant was written last-but-one. + + std::thread::sleep(Duration::from_millis(1100)); + git(&base, &["checkout", "-b", "feature"]); + fs::write( + base.join("conflict.rs"), + "fn flavour() {\n \"FEATURE_VARIANT\"\n}\n", + ) + .unwrap(); + git(&base, &["add", "conflict.rs"]); + git( + &base, + &["commit", "-m", "feature: rewrite", "--no-gpg-sign"], + ); + + std::thread::sleep(Duration::from_millis(1100)); + git(&base, &["checkout", "main"]); + fs::write( + base.join("conflict.rs"), + "fn flavour() {\n \"MAIN_VARIANT\"\n}\n", + ) + .unwrap(); + git(&base, &["add", "conflict.rs"]); + git(&base, &["commit", "-m", "main: rewrite", "--no-gpg-sign"]); + + // Also sleep before the merge itself so the post-merge worktree + // write lands in a fresh second. + std::thread::sleep(Duration::from_millis(1100)); + + // Let the divergent commits settle in the picker before we merge. + expect_file_status( + &shared_picker, + &base, + "conflict.rs", + |s| { + // Post-commit: should be clean (CURRENT or None = no row). + s.is_none() || s.unwrap().is_empty() || s.unwrap().contains(Status::CURRENT) + }, + Duration::from_secs(10), + "pre-merge clean", + ) + .expect("pre-merge state should be clean"); + + // ───────────────────────────────────────────────────────────────── + // Stage 2: trigger the conflicting merge + // ───────────────────────────────────────────────────────────────── + + // `git merge feature` returns non-zero on conflict — that's fine. + // We don't use `git(..)` (asserts success) — conflict IS the happy + // path for this test. + let merge = git_output(&base, &["merge", "feature", "--no-edit", "--no-gpg-sign"]) + .expect("git merge didn't launch"); + assert!( + !merge.status.success(), + "expected `git merge feature` to conflict, but it succeeded:\nstdout:{}\nstderr:{}", + String::from_utf8_lossy(&merge.stdout), + String::from_utf8_lossy(&merge.stderr) + ); + + // libgit2 marks both sides' modifications with `CONFLICTED`. We + // wait for the picker to match. + expect_file_status( + &shared_picker, + &base, + "conflict.rs", + |s| s.is_some_and(|st| st.contains(Status::CONFLICTED)), + CONFLICT_CONVERGE_TIMEOUT, + "CONFLICTED after merge", + ) + .expect("picker must surface CONFLICTED after merge"); + + // ───────────────────────────────────────────────────────────────── + // Stage 3: real-query surfaces must still work in conflict state + // ───────────────────────────────────────────────────────────────── + + // Fuzzy search by stem returns the conflicted file. + let fuzzy_hits = fuzzy_search_items(&shared_picker, "conflict"); + assert!( + fuzzy_hits.iter().any(|(p, _)| p == "conflict.rs"), + "fuzzy_search(\"conflict\") during conflict state returned: {:?}", + fuzzy_hits + ); + + // Grep for the diff conflict marker `<<<<<<<` — it's literally in + // the worktree file right now, so live grep must find it. + // The watcher rewrites the file on-disk during merge, so the mmap + // cache needs to have been invalidated for this to succeed. + let grep_hits = grep_plain_matches(&shared_picker, "<<<<<<< "); + assert!( + grep_hits.contains(&"conflict.rs".to_string()), + "grep(\"<<<<<<< \") during conflict state returned: {:?}", + grep_hits + ); + + // ───────────────────────────────────────────────────────────────── + // Stage 4: resolve + commit, expect return to clean + // ───────────────────────────────────────────────────────────────── + + // `on_create_or_modify` invalidates the mmap cache only when `mtime` + // advances, and mtime has 1 s resolution on every filesystem we + // ship on. `git merge` in stage 2 just wrote to `conflict.rs`; if + // we re-write it within the same second the cache keeps its + // pre-resolve bytes (complete with `<<<<<<<` markers) and grep will + // keep finding them. Sleep past the mtime tick before re-writing. + // This matches the pattern in `fuzz_file_operations.rs`. + std::thread::sleep(Duration::from_millis(1100)); + + fs::write( + base.join("conflict.rs"), + "fn flavour() {\n \"RESOLVED_VARIANT\"\n}\n", + ) + .unwrap(); + git(&base, &["add", "conflict.rs"]); + git( + &base, + &[ + "commit", + "-m", + "resolve merge", + "--no-gpg-sign", + "--no-edit", + ], + ); + + expect_file_status( + &shared_picker, + &base, + "conflict.rs", + |s| s.is_none() || s.unwrap().is_empty() || s.unwrap().contains(Status::CURRENT), + CONFLICT_CONVERGE_TIMEOUT, + "CURRENT after resolve", + ) + .expect("picker must converge back to CURRENT after conflict resolution"); + + // Sanity: conflict markers are gone from both the worktree AND the + // picker's view (grep for `<<<<<<<` now returns 0 hits). + // + // The worktree check is an independent truth source — if it fails, + // the test harness itself is buggy (the resolve write didn't apply). + // The picker check can lag briefly: `expect_file_status` returned + // once `.git/index` events made git_status = CURRENT, but the + // `conflict.rs` Modify event from our resolve `fs::write` can land + // in a separate debounced batch that hasn't been processed yet. + // Poll until the mmap cache is flushed rather than asserting once. + let on_disk = fs::read_to_string(base.join("conflict.rs")).unwrap(); + assert!( + !on_disk.contains("<<<<<<<"), + "worktree still has conflict markers after resolve — test harness bug\n\ + on-disk content:\n{on_disk}" + ); + let deadline = Instant::now() + CONFLICT_CONVERGE_TIMEOUT; + loop { + let grep_hits = grep_plain_matches(&shared_picker, "<<<<<<< "); + if !grep_hits.contains(&"conflict.rs".to_string()) { + break; + } + if Instant::now() >= deadline { + panic!( + "picker grep(\"<<<<<<< \") after resolve still returns conflict.rs \ + after {} — mmap/overlay cache is stale\n\ + (worktree on disk has no conflict markers — this is a real \ + content-invalidation bug)\n\ + last grep hits: {:?}", + humantime(CONFLICT_CONVERGE_TIMEOUT), + grep_hits, + ); + } + std::thread::sleep(CONVERGE_POLL); + } +} + +/// Seed a repo with a single file that we'll later produce a conflict in. +fn seed_conflict_repo(base: &Path) { + fs::write(base.join("README.md"), "# merge conflict test\n").unwrap(); + fs::write( + base.join("conflict.rs"), + "fn flavour() {\n \"BASE\"\n}\n", + ) + .unwrap(); + git(base, &["init", "-b", "main"]); + git(base, &["config", "user.email", "fuzz@fff.test"]); + git(base, &["config", "user.name", "fuzz"]); + // Conflict behaviour must be deterministic regardless of merge.tool. + git(base, &["config", "merge.conflictstyle", "merge"]); + git(base, &["add", "-A"]); + git(base, &["commit", "-m", "seed", "--no-gpg-sign"]); +} + +/// Poll the picker for `relative` until `predicate` holds on its +/// `git_status`, or the timeout expires. On expiry returns an `Err` with +/// the last observed status + truth status for diagnostics. +fn expect_file_status( + shared: &SharedFilePicker, + base: &Path, + relative: &str, + predicate: impl Fn(Option) -> bool, + timeout: Duration, + what: &str, +) -> Result<(), String> { + let deadline = Instant::now() + timeout; + let mut last_picker; + let mut last_truth; + loop { + let picker_status = probe_single_file_status(shared, relative).flatten(); + let truth_status = read_truth_status(base).get(relative).copied(); + last_picker = picker_status; + last_truth = truth_status; + if predicate(picker_status) { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "timed out after {} waiting for `{relative}` to satisfy `{what}`\n\ + last picker status: {:?}\n\ + last truth status : {:?}", + humantime(timeout), + last_picker, + last_truth, + )); + } + std::thread::sleep(CONVERGE_POLL); + } +} diff --git a/crates/fff-core/tests/fuzz_real_repos.proptest-regressions b/crates/fff-core/tests/fuzz_real_repos.proptest-regressions new file mode 100644 index 0000000..ae5f635 --- /dev/null +++ b/crates/fff-core/tests/fuzz_real_repos.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 2bd1032403c66aac8b9bc62a50fe04cdc6a4e65e7f00c0f8cd99839a9469f3ba # shrinks to ops = [EditRandom { seed: 3973979185 }, IgnoredBurst { count: 19, seed: 511160782 }, EditTracked { seed: 985198351 }, IgnoredBurst { count: 3, seed: 1322513110 }, IgnoredBurst { count: 8, seed: 1671662344 }, EditTracked { seed: 1561297312 }, Verify, DeleteTracked, RevertTracked, EditRandom { seed: 579281187 }, IgnoredBurst { count: 2, seed: 3742469221 }, EditRandom { seed: 3676092681 }, EditRandom { seed: 2193336478 }, EditTracked { seed: 3160375285 }, Verify, DeleteTracked, CreateFile { seed: 2245201073 }, EditRandom { seed: 1752067380 }, EditTracked { seed: 3600913031 }, EditTracked { seed: 1180012156 }, DeleteTracked, RevertTracked, IgnoredBurst { count: 11, seed: 4074832553 }, Verify, EditRandom { seed: 89716519 }, EditRandom { seed: 2578352041 }, RevertTracked, CreateFile { seed: 831766394 }, DeleteTracked, EditTracked { seed: 2532322292 }, Verify, EditTracked { seed: 3896368397 }, DeleteTracked, Verify, EditRandom { seed: 239317770 }, EditRandom { seed: 3784639684 }, EditRandom { seed: 3020447078 }, RevertTracked, CreateFile { seed: 2240840307 }, Verify, EditRandom { seed: 644157265 }, Verify, CreateFile { seed: 650224092 }, EditTracked { seed: 4211001586 }, Verify, CreateFile { seed: 594290450 }, RevertTracked, Verify, Verify, EditTracked { seed: 1256892446 }, EditRandom { seed: 2896781441 }, CreateFile { seed: 3744921304 }, Verify, Verify, RevertTracked, EditRandom { seed: 3929541705 }, RevertTracked, CreateFile { seed: 2225889515 }, IgnoredBurst { count: 4, seed: 296850726 }, EditRandom { seed: 1925610793 }, EditRandom { seed: 3903819872 }, EditRandom { seed: 3948605713 }, EditRandom { seed: 590113476 }, EditTracked { seed: 3088835824 }, EditTracked { seed: 1669401121 }, EditTracked { seed: 3243754165 }, IgnoredBurst { count: 4, seed: 3552112106 }, EditTracked { seed: 4218311739 }, RevertTracked, DeleteTracked, EditRandom { seed: 2695626522 }, Verify, DeleteTracked, Verify, Verify, CreateFile { seed: 1129013495 }, EditTracked { seed: 2692665451 }, EditRandom { seed: 1936337204 }, EditTracked { seed: 3314964290 }, CreateFile { seed: 479382716 }, Verify, DeleteTracked, Verify, CreateFile { seed: 2200064089 }, EditRandom { seed: 1120920528 }, EditTracked { seed: 12145115 }, EditTracked { seed: 305976052 }, RevertTracked, RevertTracked, DeleteTracked, CreateFile { seed: 3971122115 }, RevertTracked, DeleteTracked, DeleteTracked, Verify, RevertTracked, CreateFile { seed: 357392470 }, Verify, DeleteTracked, EditTracked { seed: 3744458445 }, CreateFile { seed: 269948916 }, RevertTracked, DeleteTracked, EditRandom { seed: 2375982740 }, CreateFile { seed: 4078337323 }, EditTracked { seed: 991718099 }, DeleteTracked, EditRandom { seed: 3561974390 }, Verify, EditTracked { seed: 1719995704 }, Verify, EditTracked { seed: 1455984559 }, CreateFile { seed: 3576941322 }, Verify, Verify, Verify, RevertTracked, EditTracked { seed: 2875597788 }, IgnoredBurst { count: 7, seed: 3483740109 }, Verify, RevertTracked, EditRandom { seed: 4148423322 }, Verify, IgnoredBurst { count: 10, seed: 403314845 }, EditTracked { seed: 426978255 }, EditRandom { seed: 2231943361 }, EditRandom { seed: 3794886001 }, CreateFile { seed: 405760745 }, EditRandom { seed: 2787015738 }, DeleteTracked, EditTracked { seed: 4162235798 }, CreateFile { seed: 54301828 }, EditTracked { seed: 257320116 }, EditTracked { seed: 2968447638 }, RevertTracked, EditRandom { seed: 2281122562 }, EditTracked { seed: 3848349126 }, EditTracked { seed: 2392477458 }, CreateFile { seed: 578024549 }, Verify, EditTracked { seed: 3148899221 }, Verify, EditTracked { seed: 3039340310 }, Verify, DeleteTracked, IgnoredBurst { count: 19, seed: 710316948 }, EditRandom { seed: 1706703248 }, Verify, EditTracked { seed: 1192341874 }, EditTracked { seed: 1921337292 }, Verify, IgnoredBurst { count: 3, seed: 1595406682 }, Verify, IgnoredBurst { count: 17, seed: 1007383845 }, CreateFile { seed: 3522710380 }, EditTracked { seed: 469984332 }, DeleteTracked, IgnoredBurst { count: 18, seed: 2443321062 }, EditTracked { seed: 1035896127 }, RevertTracked, EditRandom { seed: 1055440338 }, Verify, Verify, EditRandom { seed: 763883156 }, Verify, CreateFile { seed: 1137224999 }, EditRandom { seed: 1633958196 }, EditTracked { seed: 3738737459 }, Verify, DeleteTracked, CreateFile { seed: 2788340683 }, Verify, RevertTracked, EditTracked { seed: 1909221278 }, Verify, IgnoredBurst { count: 17, seed: 3499099003 }, Verify, EditRandom { seed: 1144453800 }, RevertTracked, CreateFile { seed: 2109075736 }, EditTracked { seed: 712682621 }, IgnoredBurst { count: 12, seed: 3721302250 }, DeleteTracked, RevertTracked, EditRandom { seed: 4226785671 }, RevertTracked, IgnoredBurst { count: 11, seed: 219666167 }, Verify, EditTracked { seed: 2098222640 }, Verify, IgnoredBurst { count: 3, seed: 3816911240 }, EditRandom { seed: 2863901373 }, EditTracked { seed: 60488794 }, CreateFile { seed: 1367832198 }, EditTracked { seed: 1458745135 }, EditRandom { seed: 1049137857 }, Verify, EditRandom { seed: 4180835914 }, DeleteTracked, EditRandom { seed: 2165041122 }, Verify, Verify, EditRandom { seed: 222010849 }, EditTracked { seed: 2550494177 }, Verify, Verify, RevertTracked, EditTracked { seed: 434134171 }, Verify, EditRandom { seed: 395799315 }, IgnoredBurst { count: 6, seed: 4071480810 }, EditRandom { seed: 3074820789 }, EditRandom { seed: 124124793 }, EditRandom { seed: 1040834453 }, EditTracked { seed: 3416392407 }, EditRandom { seed: 3536050357 }, Verify, EditRandom { seed: 2878426631 }, IgnoredBurst { count: 9, seed: 4182247654 }, CreateFile { seed: 3190392188 }, Verify, IgnoredBurst { count: 12, seed: 315132273 }, RevertTracked, CreateFile { seed: 39441597 }, Verify, EditTracked { seed: 3827879271 }, Verify, EditTracked { seed: 3359133055 }, CreateFile { seed: 3105787734 }, IgnoredBurst { count: 7, seed: 3183850029 }, EditRandom { seed: 3334300529 }, EditTracked { seed: 1744613429 }, RevertTracked, EditRandom { seed: 1573019480 }, CreateFile { seed: 652479131 }, Verify, RevertTracked, RevertTracked, EditRandom { seed: 3675395215 }, DeleteTracked, Verify, Verify, RevertTracked, EditRandom { seed: 2541437719 }, DeleteTracked, EditRandom { seed: 3037544534 }, EditTracked { seed: 2302622735 }, EditRandom { seed: 1812723857 }, RevertTracked, DeleteTracked, EditRandom { seed: 4043600818 }, Verify, IgnoredBurst { count: 16, seed: 44679182 }, Verify, Verify, DeleteTracked, CreateFile { seed: 3008740222 }, DeleteTracked, EditRandom { seed: 2914149802 }, EditRandom { seed: 667956316 }, CreateFile { seed: 3617631237 }, DeleteTracked, EditRandom { seed: 1362877831 }, DeleteTracked, EditRandom { seed: 3523649677 }, DeleteTracked, Verify, CreateFile { seed: 1403756910 }, Verify, DeleteTracked, EditRandom { seed: 2820228808 }, CreateFile { seed: 53108399 }, EditRandom { seed: 2889329358 }, EditTracked { seed: 4292504954 }, RevertTracked, Verify, Verify, EditRandom { seed: 2606229962 }, EditRandom { seed: 1857034562 }, EditRandom { seed: 3320336332 }, CreateFile { seed: 339796935 }, EditTracked { seed: 422591184 }, Verify, IgnoredBurst { count: 2, seed: 2871058542 }, CreateFile { seed: 1314463008 }, Verify, RevertTracked, EditTracked { seed: 1598357304 }, EditRandom { seed: 2158855095 }, Verify, Verify, EditTracked { seed: 1075525192 }, EditTracked { seed: 3948788493 }, EditTracked { seed: 676679568 }, Verify, Verify, EditTracked { seed: 3915678914 }, EditTracked { seed: 933469287 }, EditTracked { seed: 322191859 }, RevertTracked, CreateFile { seed: 1571061118 }, EditTracked { seed: 3844952748 }, EditTracked { seed: 3965010100 }, EditTracked { seed: 1584126813 }, Verify, EditRandom { seed: 17391951 }, Verify, IgnoredBurst { count: 14, seed: 3720249162 }, EditTracked { seed: 1131322973 }, RevertTracked, Verify, EditTracked { seed: 430594834 }, CreateFile { seed: 2763590962 }, Verify, IgnoredBurst { count: 15, seed: 1426969464 }, Verify, IgnoredBurst { count: 15, seed: 3646366033 }, EditTracked { seed: 552179910 }, CreateFile { seed: 1921856193 }, CreateFile { seed: 1102669100 }, EditTracked { seed: 3277958796 }, IgnoredBurst { count: 13, seed: 1012655776 }, Verify, EditRandom { seed: 427753982 }, CreateFile { seed: 3334360159 }, EditTracked { seed: 340831448 }, EditTracked { seed: 2131800022 }, EditRandom { seed: 2353283979 }, EditRandom { seed: 3450629600 }, Verify, Verify, EditRandom { seed: 4139689825 }, EditTracked { seed: 682373982 }, IgnoredBurst { count: 8, seed: 2882909824 }, Verify, IgnoredBurst { count: 19, seed: 4002673086 }, EditRandom { seed: 2715554126 }, DeleteTracked, EditRandom { seed: 4179922296 }, Verify, EditTracked { seed: 109690251 }, DeleteTracked, EditTracked { seed: 3472667504 }, EditRandom { seed: 1714341831 }, EditRandom { seed: 667083687 }, Verify, Verify, IgnoredBurst { count: 10, seed: 3128910574 }, CreateFile { seed: 2540247207 }, Verify, IgnoredBurst { count: 10, seed: 1511807149 }, RevertTracked, EditTracked { seed: 78824091 }, DeleteTracked, EditTracked { seed: 1898400293 }, EditRandom { seed: 3755754295 }, EditRandom { seed: 4121118104 }, EditRandom { seed: 432269839 }, EditRandom { seed: 145482715 }, EditRandom { seed: 3236540540 }, EditRandom { seed: 565508118 }, EditTracked { seed: 825926783 }, Verify, Verify, Verify, EditTracked { seed: 3586835935 }, EditRandom { seed: 2740340414 }, Verify, Verify, EditRandom { seed: 3940654502 }, RevertTracked, DeleteTracked, EditRandom { seed: 2108439768 }, IgnoredBurst { count: 2, seed: 1508577991 }, Verify, Verify, EditRandom { seed: 184009805 }, EditTracked { seed: 997928525 }, IgnoredBurst { count: 6, seed: 3249975968 }, Verify, EditRandom { seed: 2001160028 }, CreateFile { seed: 108745163 }, EditRandom { seed: 2863438947 }, EditTracked { seed: 4080526352 }, IgnoredBurst { count: 16, seed: 3652136205 }, EditRandom { seed: 33573748 }, IgnoredBurst { count: 1, seed: 2202984205 }, EditRandom { seed: 87597533 }, Verify, IgnoredBurst { count: 7, seed: 2388341587 }, Verify, Verify, EditRandom { seed: 2790460089 }, CreateFile { seed: 450719257 }, Verify, Verify, EditRandom { seed: 1611849709 }, CreateFile { seed: 1223048280 }, Verify, Verify, CreateFile { seed: 925249148 }, Verify, DeleteTracked, CreateFile { seed: 370097360 }, Verify, IgnoredBurst { count: 8, seed: 2407036983 }, EditRandom { seed: 1564783191 }, RevertTracked, Verify, CreateFile { seed: 3474431609 }, CreateFile { seed: 2854085921 }, EditRandom { seed: 1850806274 }, Verify, CreateFile { seed: 4017183086 }, IgnoredBurst { count: 1, seed: 3017226653 }, DeleteTracked, DeleteTracked, RevertTracked, CreateFile { seed: 3267789236 }, EditRandom { seed: 1402969627 }, CreateFile { seed: 402825408 }, CreateFile { seed: 243351288 }, EditRandom { seed: 4086766584 }, EditTracked { seed: 2303712775 }, EditRandom { seed: 2539341008 }, EditRandom { seed: 2943360338 }, Verify, Verify, Verify, EditTracked { seed: 1678472286 }, IgnoredBurst { count: 18, seed: 1113256590 }, EditTracked { seed: 3844639989 }, EditRandom { seed: 3152676663 }, EditTracked { seed: 1805905969 }, Verify, IgnoredBurst { count: 9, seed: 290834906 }, EditRandom { seed: 567725288 }, EditTracked { seed: 3191796340 }, CreateFile { seed: 1367118511 }, EditRandom { seed: 3207404862 }, CreateFile { seed: 113773843 }, Verify, DeleteTracked, Verify, Verify, RevertTracked, RevertTracked, Verify, Verify, CreateFile { seed: 2149772306 }, EditTracked { seed: 2188875472 }, IgnoredBurst { count: 1, seed: 1153978335 }, EditTracked { seed: 2377669181 }, Verify, DeleteTracked, Verify, EditRandom { seed: 949191429 }, EditTracked { seed: 3358882450 }, CreateFile { seed: 3525359493 }, Verify, EditRandom { seed: 2805471848 }, CreateFile { seed: 352194141 }, EditRandom { seed: 2734771285 }, EditTracked { seed: 1609339806 }, Verify, DeleteTracked, RevertTracked, IgnoredBurst { count: 7, seed: 394177225 }, RevertTracked, EditRandom { seed: 707035856 }, EditRandom { seed: 1163161576 }, EditRandom { seed: 3784310466 }, Verify, IgnoredBurst { count: 14, seed: 352157784 }, DeleteTracked, CreateFile { seed: 3671052303 }, Verify, EditTracked { seed: 768235780 }, Verify, Verify, Verify, RevertTracked, Verify, IgnoredBurst { count: 11, seed: 1652987278 }, EditRandom { seed: 1690752738 }, CreateFile { seed: 3889196883 }, EditRandom { seed: 3861916787 }, EditRandom { seed: 3494186120 }, EditRandom { seed: 2435178635 }, Verify, EditTracked { seed: 4194577038 }, CreateFile { seed: 1759771201 }, Verify, EditRandom { seed: 2205234621 }, CreateFile { seed: 1967348885 }, RevertTracked, DeleteTracked, EditTracked { seed: 311618443 }, IgnoredBurst { count: 9, seed: 1690509918 }, RevertTracked, EditRandom { seed: 2772778767 }, EditRandom { seed: 681531201 }, EditTracked { seed: 2951485348 }, Verify, Verify, EditTracked { seed: 1236848693 }, Verify, Verify, Verify, EditTracked { seed: 3790271369 }, CreateFile { seed: 297372231 }, CreateFile { seed: 55480484 }, CreateFile { seed: 2770541116 }, Verify, DeleteTracked, EditRandom { seed: 1397524124 }, IgnoredBurst { count: 9, seed: 1957554657 }, DeleteTracked, Verify, EditRandom { seed: 3150743676 }, Verify, Verify, CreateFile { seed: 3019311451 }, RevertTracked, Verify, EditTracked { seed: 1315394490 }, Verify, DeleteTracked, EditRandom { seed: 2728101238 }, Verify, CreateFile { seed: 2614566715 }, Verify, CreateFile { seed: 2856960945 }, CreateFile { seed: 2079561091 }, CreateFile { seed: 2874265061 }, IgnoredBurst { count: 14, seed: 2194472929 }, DeleteTracked, EditRandom { seed: 2355490907 }, EditTracked { seed: 2265313504 }, EditTracked { seed: 3433250532 }, RevertTracked, EditRandom { seed: 2310360272 }, EditTracked { seed: 3073533473 }, EditRandom { seed: 2761103227 }, EditRandom { seed: 1890282854 }, DeleteTracked, CreateFile { seed: 2927938188 }, EditRandom { seed: 2665233940 }, IgnoredBurst { count: 4, seed: 3140578760 }, RevertTracked, Verify, RevertTracked, Verify, EditTracked { seed: 3866635085 }, EditTracked { seed: 4188006370 }, Verify, DeleteTracked, DeleteTracked, CreateFile { seed: 703990393 }, EditRandom { seed: 1941957214 }, EditTracked { seed: 3643328766 }, RevertTracked, EditTracked { seed: 3075292415 }, EditTracked { seed: 2804574049 }, RevertTracked, EditRandom { seed: 2626551763 }, CreateFile { seed: 2386531338 }, Verify, Verify, EditRandom { seed: 1446652070 }, EditRandom { seed: 3664099977 }, IgnoredBurst { count: 4, seed: 555692681 }, EditRandom { seed: 4055639373 }, EditRandom { seed: 1739864219 }, EditRandom { seed: 131450651 }, CreateFile { seed: 2141237411 }, EditTracked { seed: 2520434680 }, CreateFile { seed: 2310992469 }, Verify, Verify, EditTracked { seed: 4033983783 }, RevertTracked, EditRandom { seed: 1681952326 }, IgnoredBurst { count: 12, seed: 1097702745 }, EditRandom { seed: 1894538168 }, EditTracked { seed: 2519177745 }, IgnoredBurst { count: 15, seed: 3900852461 }, CreateFile { seed: 4216761454 }, Verify, EditTracked { seed: 2786480369 }, EditTracked { seed: 2610087916 }, CreateFile { seed: 3452677131 }, Verify, DeleteTracked, Verify, IgnoredBurst { count: 15, seed: 3534407303 }, RevertTracked, EditRandom { seed: 2772921212 }, Verify, RevertTracked, Verify, IgnoredBurst { count: 6, seed: 2671801194 }, CreateFile { seed: 1602036010 }, EditTracked { seed: 172157022 }, Verify, Verify, EditRandom { seed: 3172236186 }, EditRandom { seed: 4287742109 }, EditRandom { seed: 2303139980 }, CreateFile { seed: 3373411723 }, EditTracked { seed: 3989385178 }, Verify, Verify, CreateFile { seed: 2288233942 }, CreateFile { seed: 649467595 }, DeleteTracked, CreateFile { seed: 2260528871 }, EditTracked { seed: 1272569946 }, RevertTracked, IgnoredBurst { count: 6, seed: 2334681169 }, Verify, IgnoredBurst { count: 9, seed: 348714819 }, IgnoredBurst { count: 19, seed: 4225071110 }, EditRandom { seed: 3340241965 }, EditRandom { seed: 1399849636 }, EditRandom { seed: 2941945108 }, EditRandom { seed: 1942382488 }, EditRandom { seed: 615375177 }, Verify, Verify, Verify, CreateFile { seed: 2534444143 }, EditRandom { seed: 3618247244 }, IgnoredBurst { count: 5, seed: 3576302117 }, EditRandom { seed: 3307892620 }, DeleteTracked, EditRandom { seed: 696665600 }, EditRandom { seed: 1140355867 }, EditTracked { seed: 600058135 }, EditRandom { seed: 3806797232 }, EditTracked { seed: 2433638424 }, CreateFile { seed: 4237702083 }, EditTracked { seed: 140970332 }, Verify, CreateFile { seed: 2239691394 }, EditRandom { seed: 1848489369 }, DeleteTracked, EditRandom { seed: 497457813 }, RevertTracked, DeleteTracked, Verify, Verify, EditTracked { seed: 277259202 }, EditTracked { seed: 1302835984 }, EditTracked { seed: 3296549517 }, DeleteTracked, DeleteTracked, EditTracked { seed: 990378599 }, EditRandom { seed: 4132447016 }, Verify, CreateFile { seed: 3273200468 }, EditRandom { seed: 3131291007 }, DeleteTracked, EditTracked { seed: 136055294 }, CreateFile { seed: 922492021 }, Verify, EditRandom { seed: 3886932299 }, DeleteTracked, Verify, IgnoredBurst { count: 11, seed: 2271499450 }, EditRandom { seed: 1336524362 }, Verify, CreateFile { seed: 3678901020 }, EditTracked { seed: 1418403844 }, RevertTracked, EditRandom { seed: 357261235 }, EditTracked { seed: 2080083529 }, CreateFile { seed: 2527380729 }, EditRandom { seed: 2698033681 }, EditRandom { seed: 3171739318 }, CreateFile { seed: 1781002527 }, EditTracked { seed: 1784082393 }, IgnoredBurst { count: 3, seed: 4019080271 }, Verify, RevertTracked, DeleteTracked, EditRandom { seed: 3732302386 }, IgnoredBurst { count: 11, seed: 2337704333 }, RevertTracked, RevertTracked, EditRandom { seed: 2745474813 }, CreateFile { seed: 1087861635 }, RevertTracked, EditRandom { seed: 550119362 }, Verify, DeleteTracked, EditTracked { seed: 2953390169 }, EditTracked { seed: 312315369 }, CreateFile { seed: 2908563734 }, DeleteTracked, EditRandom { seed: 3691372883 }, Verify, EditTracked { seed: 972966981 }, Verify, Verify, Verify, EditTracked { seed: 1626103600 }, IgnoredBurst { count: 5, seed: 2315330318 }, Verify, EditRandom { seed: 3586656111 }, CreateFile { seed: 688252447 }, EditRandom { seed: 838915923 }, Verify, RevertTracked, IgnoredBurst { count: 19, seed: 89221717 }, RevertTracked, IgnoredBurst { count: 4, seed: 1301067071 }, RevertTracked, Verify, Verify, DeleteTracked, EditTracked { seed: 64414269 }, EditTracked { seed: 1843259484 }, CreateFile { seed: 2462516468 }, RevertTracked, EditRandom { seed: 1920198788 }, IgnoredBurst { count: 11, seed: 4108018572 }, EditRandom { seed: 2411133397 }, DeleteTracked, IgnoredBurst { count: 10, seed: 3790025154 }, Verify, EditTracked { seed: 3436377394 }, EditTracked { seed: 1655435972 }, EditRandom { seed: 1392326518 }, Verify, CreateFile { seed: 3315861162 }, EditRandom { seed: 150884464 }, DeleteTracked, EditTracked { seed: 2236197900 }, CreateFile { seed: 2828168112 }, EditTracked { seed: 2665204653 }, EditTracked { seed: 1940831956 }, CreateFile { seed: 1635014651 }, Verify, Verify, CreateFile { seed: 3842785448 }, EditTracked { seed: 2119639642 }, Verify, CreateFile { seed: 3426033077 }, IgnoredBurst { count: 8, seed: 346077584 }, CreateFile { seed: 921384285 }, EditTracked { seed: 976176242 }, EditRandom { seed: 3031324220 }, Verify, EditRandom { seed: 1723898752 }, IgnoredBurst { count: 15, seed: 1981351212 }, Verify, RevertTracked, Verify, DeleteTracked, Verify, EditTracked { seed: 2638382288 }, EditTracked { seed: 1857175060 }, Verify, IgnoredBurst { count: 3, seed: 2748574242 }, RevertTracked, Verify, Verify, CreateFile { seed: 2512869688 }, EditTracked { seed: 2657306817 }, RevertTracked, CreateFile { seed: 8259479 }, IgnoredBurst { count: 15, seed: 4080748634 }, CreateFile { seed: 1155353711 }, EditRandom { seed: 2918320957 }, RevertTracked, EditTracked { seed: 991716558 }, DeleteTracked, CreateFile { seed: 809356199 }, EditTracked { seed: 1311363291 }, CreateFile { seed: 2211676489 }, Verify, DeleteTracked, Verify, Verify, EditTracked { seed: 1982412926 }, CreateFile { seed: 893631077 }, IgnoredBurst { count: 19, seed: 3870859262 }, Verify, IgnoredBurst { count: 18, seed: 3693193320 }, IgnoredBurst { count: 11, seed: 859782653 }, CreateFile { seed: 4032832134 }, Verify, Verify, Verify, CreateFile { seed: 1237648188 }, EditTracked { seed: 790755363 }, DeleteTracked, CreateFile { seed: 3774128312 }, CreateFile { seed: 3496526883 }, Verify, Verify, CreateFile { seed: 2392690045 }, EditRandom { seed: 136991493 }, EditRandom { seed: 716025343 }, EditTracked { seed: 2768444468 }, EditTracked { seed: 3767294926 }, EditTracked { seed: 3878258241 }, EditTracked { seed: 2539521037 }, Verify, EditRandom { seed: 2547660594 }, EditTracked { seed: 3759817499 }, EditRandom { seed: 1951826285 }, Verify, Verify, DeleteTracked, DeleteTracked, EditTracked { seed: 2439960371 }, DeleteTracked, Verify, EditTracked { seed: 1987151839 }, Verify, Verify, IgnoredBurst { count: 15, seed: 1291201007 }, EditTracked { seed: 3302767204 }, Verify, RevertTracked, RevertTracked, CreateFile { seed: 3841399227 }, EditRandom { seed: 1214895765 }, RevertTracked, IgnoredBurst { count: 12, seed: 4039255953 }, EditRandom { seed: 2614543638 }, CreateFile { seed: 299987642 }, Verify, Verify, Verify, RevertTracked, Verify, EditRandom { seed: 1283295384 }, RevertTracked, Verify, EditTracked { seed: 46420909 }, EditTracked { seed: 1628766460 }, CreateFile { seed: 4154494839 }, Verify, EditTracked { seed: 3373320246 }, RevertTracked, EditRandom { seed: 856537039 }, EditRandom { seed: 1817682140 }, EditRandom { seed: 2206806797 }, EditRandom { seed: 289110096 }, Verify, Verify, CreateFile { seed: 3692675069 }, EditTracked { seed: 523023608 }, Verify, Verify, Verify, IgnoredBurst { count: 10, seed: 140155456 }, EditRandom { seed: 3133447078 }, EditRandom { seed: 3319878723 }, EditTracked { seed: 4260223503 }, EditRandom { seed: 4207695806 }, CreateFile { seed: 1278913232 }, IgnoredBurst { count: 12, seed: 4225600424 }, IgnoredBurst { count: 9, seed: 2043569765 }, EditTracked { seed: 735759191 }, RevertTracked, RevertTracked, EditRandom { seed: 393746024 }, EditRandom { seed: 220590000 }, EditTracked { seed: 2851513302 }, EditTracked { seed: 1229017144 }, Verify, RevertTracked, Verify, RevertTracked, EditRandom { seed: 3936474989 }, EditRandom { seed: 2848675989 }, EditTracked { seed: 4215219303 }, IgnoredBurst { count: 11, seed: 2716022848 }, EditRandom { seed: 3984196562 }, IgnoredBurst { count: 13, seed: 3730558620 }, EditTracked { seed: 2661261439 }, EditTracked { seed: 2925705555 }, Verify, Verify, EditTracked { seed: 4021944889 }, CreateFile { seed: 1216886816 }, Verify, IgnoredBurst { count: 12, seed: 1799542122 }, DeleteTracked, EditRandom { seed: 613987795 }, CreateFile { seed: 1658013150 }, EditRandom { seed: 4140043948 }, EditRandom { seed: 765648248 }, CreateFile { seed: 2609875282 }, EditRandom { seed: 3319858271 }, CreateFile { seed: 2887454198 }, RevertTracked, EditRandom { seed: 2784515957 }, EditRandom { seed: 3602498417 }, Verify, RevertTracked, EditTracked { seed: 3923974619 }, Verify, IgnoredBurst { count: 7, seed: 330814107 }, EditTracked { seed: 3180111664 }, RevertTracked, Verify, EditTracked { seed: 2951386327 }, EditRandom { seed: 3639523727 }, CreateFile { seed: 869675643 }, Verify, EditTracked { seed: 2457361631 }, EditTracked { seed: 2131481382 }, Verify, EditTracked { seed: 976959385 }, Verify, RevertTracked, Verify, Verify, CreateFile { seed: 3046411249 }, Verify, IgnoredBurst { count: 9, seed: 4057256828 }, EditRandom { seed: 2486674547 }, Verify, EditTracked { seed: 4005659397 }, EditRandom { seed: 187353431 }, DeleteTracked, IgnoredBurst { count: 4, seed: 1705845615 }, IgnoredBurst { count: 13, seed: 715928698 }, DeleteTracked, Verify, Verify, Verify, EditTracked { seed: 434107973 }, Verify, IgnoredBurst { count: 8, seed: 1330520255 }, EditRandom { seed: 577680838 }, DeleteTracked, IgnoredBurst { count: 13, seed: 2572052106 }, CreateFile { seed: 1696727672 }, RevertTracked, EditRandom { seed: 3244995465 }, EditTracked { seed: 2800898077 }, RevertTracked, IgnoredBurst { count: 14, seed: 1565410589 }, Verify, CreateFile { seed: 2943907148 }, DeleteTracked, EditTracked { seed: 2646376363 }, Verify, EditRandom { seed: 3036192888 }, Verify, Verify, CreateFile { seed: 2469990092 }, EditRandom { seed: 2978934652 }, IgnoredBurst { count: 2, seed: 2188722887 }, EditTracked { seed: 1404626711 }, EditTracked { seed: 293022366 }, EditRandom { seed: 3512422701 }, CreateFile { seed: 2188607306 }, RevertTracked, EditTracked { seed: 2949408700 }, Verify, CreateFile { seed: 1898823355 }, RevertTracked, IgnoredBurst { count: 1, seed: 415191658 }, EditTracked { seed: 1379253955 }, CreateFile { seed: 205139567 }, DeleteTracked, Verify, EditTracked { seed: 1475569649 }, Verify, Verify, IgnoredBurst { count: 4, seed: 2175087566 }, CreateFile { seed: 1869660380 }, EditTracked { seed: 2445065907 }, EditRandom { seed: 1297287114 }, CreateFile { seed: 1591017483 }, Verify, RevertTracked, Verify, EditTracked { seed: 4250852275 }, EditRandom { seed: 3420951535 }, CreateFile { seed: 441887310 }, Verify, CreateFile { seed: 912190474 }, CreateFile { seed: 4204393654 }, CreateFile { seed: 190294792 }, EditRandom { seed: 1797322486 }, IgnoredBurst { count: 16, seed: 1983460563 }, EditRandom { seed: 2869330392 }, Verify, Verify, EditTracked { seed: 2474416608 }, EditTracked { seed: 2950234376 }, EditRandom { seed: 3016870946 }, CreateFile { seed: 3761033320 }, EditRandom { seed: 2572493295 }, Verify, DeleteTracked, DeleteTracked, CreateFile { seed: 879154639 }, RevertTracked, EditTracked { seed: 566787732 }, DeleteTracked, RevertTracked, EditRandom { seed: 1102270291 }, EditRandom { seed: 1464122371 }, IgnoredBurst { count: 11, seed: 1458095823 }, RevertTracked, EditTracked { seed: 3228026905 }, CreateFile { seed: 443749662 }, EditTracked { seed: 3761711397 }, IgnoredBurst { count: 13, seed: 2254170089 }, EditRandom { seed: 946853736 }, Verify, IgnoredBurst { count: 16, seed: 1400991987 }, DeleteTracked, Verify, EditRandom { seed: 1102454881 }, EditRandom { seed: 2965648286 }, CreateFile { seed: 2275451067 }, Verify, RevertTracked, IgnoredBurst { count: 16, seed: 4060283928 }, Verify, EditTracked { seed: 541869992 }, Verify, RevertTracked, Verify, CreateFile { seed: 978823575 }, EditRandom { seed: 1046098564 }, Verify, IgnoredBurst { count: 4, seed: 4247423185 }, RevertTracked, DeleteTracked, Verify, CreateFile { seed: 3072367945 }, IgnoredBurst { count: 5, seed: 4286542781 }, Verify, EditTracked { seed: 2185156042 }, CreateFile { seed: 2633303415 }, EditRandom { seed: 1200407260 }, RevertTracked, Verify, EditTracked { seed: 3764092935 }, Verify, EditRandom { seed: 3993565107 }, RevertTracked, IgnoredBurst { count: 8, seed: 4247583803 }, EditTracked { seed: 764064935 }, Verify, IgnoredBurst { count: 3, seed: 427540255 }, DeleteTracked, EditTracked { seed: 3200654914 }, Verify, CreateFile { seed: 3478313745 }, DeleteTracked, Verify, EditTracked { seed: 621116656 }, Verify, IgnoredBurst { count: 8, seed: 1579392476 }, Verify, EditTracked { seed: 414177081 }, Verify, EditRandom { seed: 221224112 }, EditTracked { seed: 4153706515 }, EditTracked { seed: 3045227977 }, Verify, EditRandom { seed: 2201776860 }, EditTracked { seed: 854449003 }, EditTracked { seed: 2205500297 }, EditRandom { seed: 1792458012 }, DeleteTracked, CreateFile { seed: 3424010010 }, IgnoredBurst { count: 14, seed: 1053904815 }, Verify, Verify, EditTracked { seed: 3485208950 }, DeleteTracked, Verify, EditRandom { seed: 187308236 }, EditRandom { seed: 4013451250 }, Verify, RevertTracked, IgnoredBurst { count: 6, seed: 2875702227 }, Verify, EditTracked { seed: 1296920911 }, EditTracked { seed: 291002508 }, CreateFile { seed: 506226131 }, Verify, CreateFile { seed: 3271713113 }, EditTracked { seed: 2146096155 }, Verify, EditTracked { seed: 3956111552 }, DeleteTracked, EditTracked { seed: 2909615671 }, Verify, IgnoredBurst { count: 1, seed: 3727031870 }, Verify, EditRandom { seed: 364130991 }, EditTracked { seed: 1312620454 }, RevertTracked, EditRandom { seed: 2344142345 }, Verify, RevertTracked, DeleteTracked, EditTracked { seed: 2871097942 }, Verify, EditTracked { seed: 3391171011 }, RevertTracked, CreateFile { seed: 779962761 }, EditTracked { seed: 521727725 }, IgnoredBurst { count: 11, seed: 2963591162 }, CreateFile { seed: 2729936486 }, Verify, CreateFile { seed: 656884245 }, IgnoredBurst { count: 16, seed: 276111816 }, IgnoredBurst { count: 16, seed: 1051645651 }, CreateFile { seed: 169222333 }, Verify, Verify, EditRandom { seed: 3273679269 }, Verify, Verify, EditTracked { seed: 2298741199 }, EditTracked { seed: 3460711926 }, EditTracked { seed: 2552630546 }, RevertTracked, Verify, CreateFile { seed: 2104811303 }, Verify, Verify, EditTracked { seed: 3622390374 }, EditRandom { seed: 1687431801 }, CreateFile { seed: 972620881 }, EditTracked { seed: 3470648145 }, DeleteTracked, EditTracked { seed: 2377088473 }, CreateFile { seed: 574076790 }, EditRandom { seed: 3293958247 }, EditRandom { seed: 1965906926 }, Verify, EditRandom { seed: 3684049561 }, RevertTracked, EditRandom { seed: 831878737 }, DeleteTracked, DeleteTracked, CreateFile { seed: 3771609943 }, DeleteTracked, EditTracked { seed: 3467773777 }, Verify, EditRandom { seed: 1108734627 }, EditRandom { seed: 636568845 }, CreateFile { seed: 908148475 }, EditTracked { seed: 2781804085 }, Verify, EditTracked { seed: 1537491028 }, EditTracked { seed: 3345235088 }, Verify, IgnoredBurst { count: 17, seed: 660298101 }, EditRandom { seed: 3490918256 }, EditTracked { seed: 527328431 }, EditRandom { seed: 3786143098 }, Verify, RevertTracked, Verify, EditTracked { seed: 538972755 }, CreateFile { seed: 3411292422 }, DeleteTracked, Verify, Verify, CreateFile { seed: 1670343000 }, Verify, CreateFile { seed: 3854157427 }, EditTracked { seed: 2132270146 }, EditRandom { seed: 585747811 }, EditRandom { seed: 1067438812 }, CreateFile { seed: 2842991605 }, RevertTracked, EditTracked { seed: 3437840971 }, EditTracked { seed: 1416890816 }, EditRandom { seed: 180177728 }, RevertTracked, Verify, IgnoredBurst { count: 12, seed: 196787470 }, EditRandom { seed: 926497172 }, Verify, RevertTracked, EditRandom { seed: 3016908824 }, Verify, Verify, Verify, EditRandom { seed: 2473410717 }, CreateFile { seed: 2588911915 }, DeleteTracked, IgnoredBurst { count: 16, seed: 1671443891 }, EditRandom { seed: 2899222965 }, EditRandom { seed: 3437102041 }, EditTracked { seed: 1660505442 }, Verify, CreateFile { seed: 1980030743 }, Verify, DeleteTracked, RevertTracked, EditTracked { seed: 4213383365 }, Verify, Verify, IgnoredBurst { count: 18, seed: 1866959486 }, EditRandom { seed: 906433071 }, Verify, RevertTracked, CreateFile { seed: 1354848006 }, CreateFile { seed: 2122890842 }, IgnoredBurst { count: 4, seed: 2944787547 }, EditTracked { seed: 853545580 }, RevertTracked, CreateFile { seed: 570341134 }, IgnoredBurst { count: 12, seed: 4248027101 }, EditTracked { seed: 3959338075 }, Verify, RevertTracked, EditRandom { seed: 1825554916 }, IgnoredBurst { count: 4, seed: 2413101420 }, Verify, EditRandom { seed: 3423767791 }, CreateFile { seed: 1486338580 }, EditRandom { seed: 1019557191 }, CreateFile { seed: 878976235 }, EditRandom { seed: 636852074 }, Verify, Verify, DeleteTracked, CreateFile { seed: 2946302446 }, EditTracked { seed: 836121738 }, CreateFile { seed: 1658335766 }, DeleteTracked, EditTracked { seed: 1650413622 }, Verify, EditRandom { seed: 853416845 }, CreateFile { seed: 3739132611 }, Verify, CreateFile { seed: 658006215 }, EditTracked { seed: 2210563609 }, Verify, RevertTracked, RevertTracked, EditTracked { seed: 913167404 }, IgnoredBurst { count: 18, seed: 3417417595 }, EditTracked { seed: 2520452154 }, EditTracked { seed: 2904246191 }, IgnoredBurst { count: 12, seed: 2179597638 }, RevertTracked, DeleteTracked, DeleteTracked, IgnoredBurst { count: 17, seed: 946825186 }, Verify, IgnoredBurst { count: 16, seed: 2662246565 }, IgnoredBurst { count: 4, seed: 1975237070 }, Verify, EditTracked { seed: 4125498183 }, Verify, RevertTracked, EditRandom { seed: 269387628 }, RevertTracked, EditTracked { seed: 547912836 }, EditTracked { seed: 3826598792 }, CreateFile { seed: 2534052416 }, EditTracked { seed: 1924596928 }, EditTracked { seed: 3241272543 }, EditRandom { seed: 2693571079 }, EditRandom { seed: 680678920 }, Verify, CreateFile { seed: 2929370980 }, EditRandom { seed: 62474154 }, EditTracked { seed: 2333194885 }, CreateFile { seed: 3602762209 }, DeleteTracked, DeleteTracked, RevertTracked, Verify, Verify, CreateFile { seed: 2419934392 }, EditRandom { seed: 3392361341 }, Verify, CreateFile { seed: 2315145468 }, Verify, EditRandom { seed: 4082510829 }, EditRandom { seed: 2670016769 }, CreateFile { seed: 372946636 }, IgnoredBurst { count: 1, seed: 3503961121 }, EditRandom { seed: 2623647210 }, Verify, EditRandom { seed: 613162070 }, Verify, EditTracked { seed: 392521965 }, Verify, Verify, EditRandom { seed: 3971027359 }, CreateFile { seed: 2413917220 }, EditTracked { seed: 2457512291 }, EditRandom { seed: 3045165889 }, EditRandom { seed: 846514065 }, EditTracked { seed: 1718690337 }, RevertTracked, Verify, IgnoredBurst { count: 6, seed: 2885715679 }, Verify, EditRandom { seed: 3707696231 }, RevertTracked, Verify, Verify, EditTracked { seed: 3910481912 }, RevertTracked, Verify, Verify, Verify, CreateFile { seed: 2647237027 }, CreateFile { seed: 154550653 }, Verify, DeleteTracked, EditTracked { seed: 3160016772 }, EditTracked { seed: 3777969260 }, EditRandom { seed: 386540721 }, RevertTracked, Verify, CreateFile { seed: 1246336374 }, EditTracked { seed: 2844303427 }, EditRandom { seed: 1094520810 }, Verify, Verify, CreateFile { seed: 1059724883 }, RevertTracked, Verify, CreateFile { seed: 3433364675 }, EditTracked { seed: 360509448 }, Verify, RevertTracked, RevertTracked, CreateFile { seed: 2213268263 }, Verify, Verify, Verify, Verify, EditTracked { seed: 4152429785 }, IgnoredBurst { count: 6, seed: 381853723 }, EditRandom { seed: 1962783045 }, Verify, EditRandom { seed: 1665382956 }, EditRandom { seed: 2709955475 }, RevertTracked, RevertTracked, EditTracked { seed: 3271796791 }, Verify, Verify, EditTracked { seed: 3824419611 }, EditRandom { seed: 2812898222 }, EditRandom { seed: 2689333177 }, EditRandom { seed: 122504226 }, Verify, EditTracked { seed: 928034260 }, EditRandom { seed: 3150634864 }, RevertTracked, IgnoredBurst { count: 3, seed: 138702265 }, EditTracked { seed: 1953792973 }, Verify, Verify, Verify, EditTracked { seed: 3139680678 }, DeleteTracked, CreateFile { seed: 2671266006 }, CreateFile { seed: 3798669949 }, RevertTracked, Verify, CreateFile { seed: 1751116722 }, CreateFile { seed: 1367295218 }, EditRandom { seed: 689124306 }, IgnoredBurst { count: 5, seed: 1057474153 }, EditRandom { seed: 3654782391 }, EditRandom { seed: 905410224 }, EditRandom { seed: 650131199 }, CreateFile { seed: 1644459558 }, CreateFile { seed: 3886687299 }, EditRandom { seed: 2499775116 }, RevertTracked, DeleteTracked, DeleteTracked, IgnoredBurst { count: 11, seed: 734941190 }, RevertTracked, RevertTracked, IgnoredBurst { count: 17, seed: 3297250774 }, CreateFile { seed: 825440208 }, CreateFile { seed: 2389128304 }, Verify, Verify, EditTracked { seed: 3719945320 }, Verify, CreateFile { seed: 3121352196 }, EditTracked { seed: 568124310 }, EditTracked { seed: 2547898740 }, EditRandom { seed: 664131757 }, EditRandom { seed: 2209103600 }, EditRandom { seed: 2122119482 }, Verify, IgnoredBurst { count: 3, seed: 1674489070 }, EditRandom { seed: 298454037 }, Verify, EditRandom { seed: 937087471 }, CreateFile { seed: 667119880 }, Verify, Verify, EditTracked { seed: 4140605495 }, EditRandom { seed: 907239544 }, RevertTracked, IgnoredBurst { count: 16, seed: 2754839097 }, EditTracked { seed: 2855478163 }, Verify, DeleteTracked, EditTracked { seed: 306432568 }, EditTracked { seed: 256510315 }, DeleteTracked, Verify, EditTracked { seed: 746220962 }, EditTracked { seed: 757599865 }, DeleteTracked, IgnoredBurst { count: 11, seed: 2160096413 }, EditRandom { seed: 3082716150 }, Verify, Verify, EditRandom { seed: 3620166526 }, Verify, IgnoredBurst { count: 14, seed: 2001536715 }, EditRandom { seed: 1502961340 }, EditRandom { seed: 3266625643 }, IgnoredBurst { count: 13, seed: 202899898 }, IgnoredBurst { count: 10, seed: 3407140075 }, Verify, EditRandom { seed: 1173910848 }, IgnoredBurst { count: 13, seed: 263215691 }, EditTracked { seed: 3955125598 }, EditTracked { seed: 3599717689 }, EditRandom { seed: 2927094252 }, Verify, Verify, IgnoredBurst { count: 13, seed: 35568006 }, Verify, Verify, EditTracked { seed: 2738769783 }, Verify, DeleteTracked, EditRandom { seed: 3923140387 }, DeleteTracked, EditTracked { seed: 822626989 }, Verify, Verify, EditRandom { seed: 2882342013 }, EditRandom { seed: 485956923 }, CreateFile { seed: 861742348 }, RevertTracked, EditRandom { seed: 492042665 }, EditTracked { seed: 585247444 }, EditRandom { seed: 4009444343 }, EditRandom { seed: 3860326443 }, EditRandom { seed: 246620796 }, EditRandom { seed: 1800926521 }, EditTracked { seed: 163090557 }, EditTracked { seed: 3206994326 }, Verify, RevertTracked, EditTracked { seed: 32160828 }, Verify, EditRandom { seed: 216975427 }, EditTracked { seed: 2151294415 }, Verify, Verify, IgnoredBurst { count: 17, seed: 4193562270 }, EditTracked { seed: 2346409672 }, EditRandom { seed: 2146273073 }, CreateFile { seed: 3814649567 }, EditTracked { seed: 2293264676 }, Verify, IgnoredBurst { count: 6, seed: 556908727 }, EditTracked { seed: 815136352 }, EditRandom { seed: 534097288 }, EditRandom { seed: 4206964754 }, RevertTracked, RevertTracked, CreateFile { seed: 1672388267 }, IgnoredBurst { count: 14, seed: 2106323223 }, Verify, EditRandom { seed: 4034317240 }, RevertTracked, Verify, Verify, CreateFile { seed: 664034020 }, EditTracked { seed: 2468485008 }, EditTracked { seed: 2963811379 }, IgnoredBurst { count: 4, seed: 3482405570 }, EditTracked { seed: 844543737 }, Verify, RevertTracked, CreateFile { seed: 2706694030 }, EditTracked { seed: 829300683 }, EditTracked { seed: 1825525759 }, Verify, RevertTracked, CreateFile { seed: 700629431 }, EditTracked { seed: 1020266595 }, Verify, RevertTracked, Verify, RevertTracked, Verify, IgnoredBurst { count: 6, seed: 1441230570 }, DeleteTracked, EditTracked { seed: 505269583 }, EditRandom { seed: 1762669515 }, RevertTracked, EditRandom { seed: 3932299170 }, CreateFile { seed: 202967737 }, IgnoredBurst { count: 3, seed: 853745304 }, EditTracked { seed: 3512894440 }, RevertTracked, EditRandom { seed: 1740737793 }, Verify, EditRandom { seed: 2045064956 }, IgnoredBurst { count: 12, seed: 4149572720 }, EditTracked { seed: 4201940131 }, CreateFile { seed: 1207901796 }, Verify, RevertTracked, Verify, EditTracked { seed: 951304066 }, CreateFile { seed: 2341411362 }, EditRandom { seed: 772262634 }, EditRandom { seed: 4148708385 }, IgnoredBurst { count: 13, seed: 764892389 }, EditTracked { seed: 1670413201 }, EditTracked { seed: 917463944 }, CreateFile { seed: 2795796280 }, EditRandom { seed: 2213125394 }, IgnoredBurst { count: 3, seed: 596798658 }, EditTracked { seed: 3997764374 }, EditTracked { seed: 2284564959 }, EditTracked { seed: 2178105257 }, CreateFile { seed: 1140107653 }, EditRandom { seed: 525123852 }, EditRandom { seed: 3917131157 }, EditRandom { seed: 3744551257 }, RevertTracked, CreateFile { seed: 1379408053 }, Verify, Verify, EditRandom { seed: 1081136646 }, DeleteTracked, EditTracked { seed: 1330472276 }, Verify, EditRandom { seed: 281179811 }, Verify, EditTracked { seed: 2045581787 }, DeleteTracked, EditRandom { seed: 3273155612 }, DeleteTracked, EditRandom { seed: 1695472697 }, EditRandom { seed: 257937003 }, Verify, EditRandom { seed: 3861307694 }, EditTracked { seed: 111190497 }, IgnoredBurst { count: 6, seed: 3522403470 }, EditRandom { seed: 1648818862 }, Verify, Verify, RevertTracked, Verify, IgnoredBurst { count: 11, seed: 3281563920 }, Verify, Verify, EditTracked { seed: 1584012553 }, DeleteTracked, EditRandom { seed: 1902833174 }, Verify, DeleteTracked, CreateFile { seed: 176916229 }, Verify, EditRandom { seed: 3339928290 }, Verify, CreateFile { seed: 2918766702 }, RevertTracked, EditTracked { seed: 1102055727 }, DeleteTracked, RevertTracked, EditRandom { seed: 2519140580 }, EditRandom { seed: 3257459587 }, Verify, EditRandom { seed: 4120041633 }, EditTracked { seed: 414351915 }, EditRandom { seed: 4052048038 }, EditTracked { seed: 1672775475 }, CreateFile { seed: 4045750635 }, Verify, DeleteTracked, IgnoredBurst { count: 11, seed: 2200913013 }, EditTracked { seed: 1455308882 }, DeleteTracked, Verify, DeleteTracked, EditTracked { seed: 2263201844 }, EditRandom { seed: 1649542395 }, CreateFile { seed: 2236719406 }, CreateFile { seed: 2683205360 }, RevertTracked, Verify, RevertTracked, CreateFile { seed: 594419243 }, EditRandom { seed: 3264041697 }, EditRandom { seed: 3708333463 }, CreateFile { seed: 1066002394 }, IgnoredBurst { count: 16, seed: 766551587 }, Verify, DeleteTracked, EditRandom { seed: 2912753053 }, EditTracked { seed: 1960938336 }, RevertTracked, DeleteTracked, Verify, Verify, EditRandom { seed: 469573721 }, RevertTracked, CreateFile { seed: 2411710260 }, CreateFile { seed: 2825262381 }, Verify, CreateFile { seed: 696597544 }, EditTracked { seed: 113378479 }, EditTracked { seed: 2079922246 }, Verify, EditTracked { seed: 389922696 }, EditTracked { seed: 119484527 }, RevertTracked, EditRandom { seed: 4080248257 }, EditRandom { seed: 282117367 }, RevertTracked, Verify, Verify, EditTracked { seed: 3816149941 }, RevertTracked, EditTracked { seed: 2580541526 }, EditRandom { seed: 931526831 }, DeleteTracked, IgnoredBurst { count: 2, seed: 2250707158 }, RevertTracked, EditTracked { seed: 72334749 }, DeleteTracked, CreateFile { seed: 4254515667 }, Verify, EditTracked { seed: 2977825108 }, CreateFile { seed: 2118623053 }, Verify, EditRandom { seed: 1752472528 }, DeleteTracked, Verify, EditRandom { seed: 927104512 }, Verify, Verify, EditTracked { seed: 3900220506 }, CreateFile { seed: 2507457466 }, Verify, Verify, RevertTracked, EditTracked { seed: 3828752965 }, EditRandom { seed: 3515611587 }, EditRandom { seed: 3949720206 }, RevertTracked, Verify, DeleteTracked, Verify, EditRandom { seed: 4087835278 }, DeleteTracked, Verify, Verify, Verify, RevertTracked, RevertTracked, EditRandom { seed: 596424732 }, EditTracked { seed: 1140716885 }, Verify, EditTracked { seed: 558373333 }, RevertTracked, EditTracked { seed: 2026552879 }, Verify, EditTracked { seed: 645364163 }, RevertTracked, DeleteTracked, EditTracked { seed: 4217761095 }, Verify, RevertTracked, EditRandom { seed: 2482782805 }, EditRandom { seed: 2613240720 }, EditTracked { seed: 834291878 }, EditTracked { seed: 4039197305 }, RevertTracked, RevertTracked, EditRandom { seed: 1474885024 }, EditTracked { seed: 3957783681 }, EditRandom { seed: 2642243967 }, Verify, EditTracked { seed: 3492681076 }, EditTracked { seed: 860448737 }, EditRandom { seed: 1922334772 }, Verify, CreateFile { seed: 963656693 }, EditTracked { seed: 224488887 }, CreateFile { seed: 3902106102 }, Verify, EditRandom { seed: 2723002705 }, Verify, Verify, CreateFile { seed: 2754356156 }, RevertTracked, Verify, EditRandom { seed: 1126492744 }, Verify, EditTracked { seed: 1706446609 }, Verify, EditRandom { seed: 4116800247 }, CreateFile { seed: 2387839606 }, DeleteTracked, CreateFile { seed: 1607764878 }, DeleteTracked, EditRandom { seed: 223778629 }, Verify, IgnoredBurst { count: 2, seed: 1176689445 }, EditRandom { seed: 808391150 }, CreateFile { seed: 1901698439 }, IgnoredBurst { count: 5, seed: 728641430 }, CreateFile { seed: 1907099913 }, EditTracked { seed: 2537863683 }, IgnoredBurst { count: 8, seed: 1023238415 }, EditRandom { seed: 4041706823 }, EditRandom { seed: 1602784916 }, EditTracked { seed: 1549314113 }, EditTracked { seed: 4033324071 }, EditTracked { seed: 2182470899 }, EditRandom { seed: 2276047190 }, Verify, DeleteTracked, EditRandom { seed: 927506698 }, EditTracked { seed: 3149375890 }, Verify, Verify, EditRandom { seed: 4277504923 }, RevertTracked, EditTracked { seed: 1975228644 }, CreateFile { seed: 2082003949 }, DeleteTracked, RevertTracked, EditRandom { seed: 3830488050 }, EditRandom { seed: 1076924959 }, EditTracked { seed: 1386713996 }, EditTracked { seed: 3210991949 }, EditTracked { seed: 3078541723 }, EditRandom { seed: 2343918484 }, EditRandom { seed: 3635830388 }, EditTracked { seed: 2434224085 }, EditRandom { seed: 3520012572 }, IgnoredBurst { count: 6, seed: 392731601 }, Verify, DeleteTracked, RevertTracked, RevertTracked, RevertTracked, EditTracked { seed: 1220306616 }, IgnoredBurst { count: 7, seed: 481148563 }, RevertTracked, RevertTracked, EditRandom { seed: 2546373421 }, Verify, DeleteTracked, RevertTracked, Verify, RevertTracked, Verify, Verify, IgnoredBurst { count: 4, seed: 1743420290 }, EditRandom { seed: 1671286671 }, EditRandom { seed: 3302143154 }, Verify, Verify, Verify, IgnoredBurst { count: 4, seed: 2099457533 }, IgnoredBurst { count: 14, seed: 3096449830 }, Verify, EditRandom { seed: 995587542 }, Verify, CreateFile { seed: 4239505407 }, DeleteTracked, EditTracked { seed: 3604289508 }, EditRandom { seed: 1916994892 }, IgnoredBurst { count: 13, seed: 1880112792 }, Verify, EditRandom { seed: 3422881599 }, CreateFile { seed: 484865746 }, Verify, CreateFile { seed: 3651205812 }, Verify, RevertTracked, RevertTracked, EditTracked { seed: 972766116 }, EditTracked { seed: 1207515475 }, EditRandom { seed: 3201008215 }, EditRandom { seed: 3479803416 }, IgnoredBurst { count: 10, seed: 1399322284 }, Verify, RevertTracked, EditRandom { seed: 3887992924 }, RevertTracked, EditRandom { seed: 418702839 }, Verify, RevertTracked, Verify, EditTracked { seed: 1065210807 }, Verify, RevertTracked, Verify, EditRandom { seed: 826912833 }, Verify, EditTracked { seed: 2872237440 }, CreateFile { seed: 2563313293 }, EditTracked { seed: 852514814 }, EditTracked { seed: 3478661577 }, EditTracked { seed: 3286523340 }, EditRandom { seed: 4123252795 }, EditRandom { seed: 1952651556 }, Verify, DeleteTracked, EditRandom { seed: 1527255102 }, EditRandom { seed: 3567122844 }, Verify, EditTracked { seed: 3588747821 }, CreateFile { seed: 1195654490 }, Verify, Verify, EditTracked { seed: 3472290727 }, Verify, RevertTracked, EditTracked { seed: 2941042700 }, CreateFile { seed: 2287920678 }, CreateFile { seed: 3550162613 }, EditTracked { seed: 3631399778 }, EditTracked { seed: 2128456710 }, EditRandom { seed: 3521715884 }, EditRandom { seed: 955894777 }, CreateFile { seed: 1908433814 }, EditRandom { seed: 1869362800 }, RevertTracked, EditRandom { seed: 2644729877 }, Verify, Verify, IgnoredBurst { count: 1, seed: 1175536885 }, Verify, EditRandom { seed: 2445799630 }, EditTracked { seed: 3448441725 }, Verify, RevertTracked, EditTracked { seed: 2683709987 }, IgnoredBurst { count: 2, seed: 3536509227 }, IgnoredBurst { count: 1, seed: 2527940078 }, RevertTracked, EditTracked { seed: 895354678 }, CreateFile { seed: 99616131 }, EditRandom { seed: 113656732 }, EditRandom { seed: 320283757 }, IgnoredBurst { count: 17, seed: 1979448930 }, EditRandom { seed: 1810766813 }, IgnoredBurst { count: 17, seed: 1897616006 }, Verify, EditTracked { seed: 3513974371 }, DeleteTracked, Verify, Verify, EditRandom { seed: 3520385208 }, DeleteTracked, EditRandom { seed: 2158730202 }, EditTracked { seed: 1113953409 }, DeleteTracked, Verify, EditTracked { seed: 1700902146 }, CreateFile { seed: 1512862897 }, CreateFile { seed: 2140989124 }, EditTracked { seed: 4091224770 }, RevertTracked, IgnoredBurst { count: 7, seed: 1907437660 }, CreateFile { seed: 3001916767 }, EditRandom { seed: 297303762 }, EditRandom { seed: 2311955576 }, DeleteTracked, CreateFile { seed: 3985453137 }, Verify, Verify, Verify, RevertTracked, EditRandom { seed: 1809874124 }, Verify, RevertTracked, EditTracked { seed: 2235849743 }, Verify, EditRandom { seed: 1735992863 }, RevertTracked, EditTracked { seed: 2112545184 }, EditTracked { seed: 2348992401 }, RevertTracked, Verify, RevertTracked, CreateFile { seed: 3818034267 }, Verify, EditTracked { seed: 1840036803 }, CreateFile { seed: 2403107875 }, IgnoredBurst { count: 13, seed: 2506272199 }, CreateFile { seed: 3865626955 }, EditTracked { seed: 2826614713 }, EditTracked { seed: 3363985451 }, EditRandom { seed: 1212975417 }, EditRandom { seed: 2152041037 }, EditTracked { seed: 793681097 }, EditTracked { seed: 1997627309 }, EditTracked { seed: 3071923675 }, Verify, EditTracked { seed: 1799630046 }, EditRandom { seed: 884045638 }, CreateFile { seed: 1145072754 }, EditRandom { seed: 3990868217 }, Verify, EditTracked { seed: 115769826 }, DeleteTracked, CreateFile { seed: 1294002809 }, CreateFile { seed: 4127199573 }, RevertTracked, Verify, EditTracked { seed: 2704632093 }, IgnoredBurst { count: 11, seed: 1756693692 }, Verify, IgnoredBurst { count: 5, seed: 1127075575 }, Verify, Verify, EditTracked { seed: 240115701 }, Verify, Verify, EditRandom { seed: 349322801 }, EditRandom { seed: 621026824 }, RevertTracked, IgnoredBurst { count: 4, seed: 1287979947 }, Verify, Verify, Verify, Verify, EditRandom { seed: 249751148 }, EditTracked { seed: 57306234 }, Verify, RevertTracked, RevertTracked, IgnoredBurst { count: 10, seed: 4248317531 }, EditRandom { seed: 356090260 }, EditTracked { seed: 2360822711 }, Verify, DeleteTracked, CreateFile { seed: 2579991617 }, EditRandom { seed: 3746166339 }, Verify, EditTracked { seed: 1149677200 }, EditTracked { seed: 1908760634 }, EditRandom { seed: 422070904 }, CreateFile { seed: 2276869479 }, EditRandom { seed: 2352413156 }, RevertTracked, EditRandom { seed: 1730733757 }, IgnoredBurst { count: 12, seed: 3274703388 }, IgnoredBurst { count: 18, seed: 1187502331 }, RevertTracked, Verify, Verify, CreateFile { seed: 3281095308 }, Verify, EditTracked { seed: 100517685 }, DeleteTracked, EditRandom { seed: 2857109687 }, DeleteTracked, EditRandom { seed: 3573073694 }, EditRandom { seed: 2588775720 }, EditTracked { seed: 3409550332 }, RevertTracked, EditRandom { seed: 3476087259 }, CreateFile { seed: 1810301050 }, DeleteTracked, CreateFile { seed: 4248530530 }, RevertTracked, Verify, CreateFile { seed: 4082784410 }] diff --git a/crates/fff-core/tests/fuzz_real_repos.rs b/crates/fff-core/tests/fuzz_real_repos.rs new file mode 100644 index 0000000..b06851a --- /dev/null +++ b/crates/fff-core/tests/fuzz_real_repos.rs @@ -0,0 +1,712 @@ +//! Proptest-driven fuzz test against real GitHub repos with a live watcher. +//! +//! Clones real repository, runs the simulated close to real user sereies of file system ewvents and +//! verifies that fff can still find the correct files. Test cases are randomized and preserved +//! using proptest +#![cfg(stress)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use proptest::prelude::*; +use proptest::test_runner::{Config as ProptestConfig, FileFailurePersistence}; + +use fff_search::file_picker::{FFFMode, FilePicker, is_known_binary_extension}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency}; + +const REPO_POOL: &[(&str, &str)] = &[ + ("dmtrKovalenko/fff", "fff"), + ("BurntSushi/ripgrep", "ripgrep"), + ("sharkdp/fd", "fd"), + ("ogham/exa", "exa"), + ("casey/just", "just"), + ("ajeetdsouza/zoxide", "zoxide"), + ("helix-editor/helix", "helix"), + ("astral-sh/ruff", "ruff"), + ("biomejs/biome", "biome"), + ("denoland/deno_lint", "deno_lint"), + ("nickel-lang/nickel", "nickel"), + ("typst/typst", "typst"), + ("gleam-lang/gleam", "gleam"), + ("pretzelhammer/rust-blog", "rust-blog"), + ("tokio-rs/mini-redis", "mini-redis"), +]; + +const CACHE_DIR: &str = "/tmp/fff_fuzz_repos"; +/// Fixed settle time for watcher event propagation. +const WATCHER_SETTLE: Duration = Duration::from_millis(100); +/// Maximum time to wait for watcher to process all pending events. +const CONVERGE_TIMEOUT: Duration = Duration::from_secs(30); + +fn fuzz_cases() -> u32 { + std::env::var("FFF_FUZZ_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2) +} + +fn fuzz_max_ops() -> usize { + std::env::var("FFF_FUZZ_MAX_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(30) +} + +fn fuzz_min_ops() -> usize { + std::env::var("FFF_FUZZ_MIN_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(15) +} + +fn ensure_repo_cloned(repo_url: &str, local_name: &str) -> PathBuf { + let cache = PathBuf::from(CACHE_DIR); + fs::create_dir_all(&cache).unwrap(); + let repo_path = cache.join(local_name); + if repo_path.join(".git").exists() { + return repo_path; + } + + let full_url = format!("https://github.com/{}.git", repo_url); + eprintln!(" Cloning {} ...", full_url); + let out = Command::new("git") + .args(["clone", "--depth=1", "--single-branch", &full_url]) + .arg(&repo_path) + .output() + .expect("git clone failed"); + assert!( + out.status.success(), + "git clone {} failed: {}", + full_url, + String::from_utf8_lossy(&out.stderr) + ); + repo_path +} + +fn copy_repo_to_workdir(cached: &Path, workdir: &Path) { + let out = Command::new("cp") + .args(["-r"]) + .arg(cached) + .arg(workdir) + .output() + .expect("cp -r failed"); + assert!( + out.status.success(), + "cp -r failed: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +fn collect_text_files(base: &Path) -> Vec { + // Use `git ls-files` without --cached to get only files that are both + // tracked AND not gitignored. Files like Cargo.lock that are committed + // but in .gitignore would appear with --cached but the fff picker skips + // them during walk (respects .gitignore), causing false test failures. + let out = Command::new("git") + .args(["ls-files", "--others", "--exclude-standard", "-z"]) + .current_dir(base) + .output() + .unwrap(); + + // Get tracked files that aren't ignored + let tracked = Command::new("git") + .args(["ls-files", "-z"]) + .current_dir(base) + .output() + .unwrap(); + + // Check which tracked files are actually ignored + let ignored_check = Command::new("git") + .args(["check-ignore", "--stdin", "-z"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .current_dir(base) + .spawn(); + + let mut ignored_set: std::collections::HashSet = std::collections::HashSet::new(); + if let Ok(mut child) = ignored_check { + use std::io::Write; + if let Some(ref mut stdin) = child.stdin { + let _ = stdin.write_all(&tracked.stdout); + } + if let Ok(output) = child.wait_with_output() { + for path in output.stdout.split(|&b| b == 0) { + if !path.is_empty() { + if let Ok(s) = std::str::from_utf8(path) { + ignored_set.insert(s.to_string()); + } + } + } + } + } + + // Combine: tracked non-ignored non-binary files + let mut files: Vec = Vec::new(); + for path in tracked.stdout.split(|&b| b == 0) { + if path.is_empty() { + continue; + } + let Ok(s) = std::str::from_utf8(path) else { + continue; + }; + if ignored_set.contains(s) { + continue; + } + let full = base.join(s); + if full.is_file() && !is_known_binary_extension(&full) { + files.push(full); + } + } + files +} + +/// Edit a file by injecting a marker line at a deterministic position, +/// preserving the rest of the content. Returns the original line that was +/// replaced so it can be restored on revert. +fn inject_marker(path: &Path, marker: &str, seed: u32) -> Option { + let content = fs::read_to_string(path).ok()?; + let lines: Vec<&str> = content.lines().collect(); + if lines.is_empty() { + fs::write(path, format!("// {marker}\n")).ok()?; + return Some(String::new()); + } + + // Pick a stable line position based on seed and file length + let line_idx = seed as usize % lines.len(); + let original_line = lines[line_idx].to_string(); + + let mut result = String::with_capacity(content.len() + marker.len() + 10); + for (i, line) in lines.iter().enumerate() { + if i == line_idx { + result.push_str(&format!("// {marker}")); + } else { + result.push_str(line); + } + result.push('\n'); + } + fs::write(path, &result).ok()?; + Some(original_line) +} + +/// Revert a file by restoring the original line at the same position +/// where inject_marker placed the marker. +fn revert_marker(path: &Path, marker: &str, original_line: &str) { + let Ok(content) = fs::read_to_string(path) else { + return; + }; + let marker_line = format!("// {marker}"); + let result: String = content + .lines() + .map(|l| if l == marker_line { original_line } else { l }) + .collect::>() + .join("\n") + + "\n"; + let _ = fs::write(path, result); +} + +fn grep_opts(mode: GrepMode) -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode, + time_budget_ms: 5000, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +fn grep_finds(picker: &FilePicker, query: &str, mode: GrepMode) -> bool { + let parsed = parse_grep_query(query); + let result = picker.grep(&parsed, &grep_opts(mode)); + !result.matches.is_empty() +} + +fn grep_file_list(picker: &FilePicker, query: &str, mode: GrepMode) -> Vec { + let parsed = parse_grep_query(query); + let result = picker.grep(&parsed, &grep_opts(mode)); + result + .files + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Infrastructure +// ═══════════════════════════════════════════════════════════════════════════ + +fn wait_for_bigram(sp: &SharedFilePicker) { + let deadline = Instant::now() + Duration::from_secs(120); + loop { + std::thread::sleep(Duration::from_millis(50)); + let ready = sp + .read() + .ok() + .map(|g| { + g.as_ref() + .map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + return; + } + assert!( + Instant::now() < deadline, + "Timed out waiting for bigram index" + ); + } +} + +fn epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +struct TrackedFile { + relative: String, + marker: String, + /// The original line content that was replaced, for revert + original_line: String, + is_created: bool, + last_write_sec: u64, +} + +fn run_scenario(ops: &[Op]) { + // Stream fff logs at info+ level by default. Override with RUST_LOG. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn,fff_search=info")), + ) + .with_test_writer() + .try_init(); + + // Allow forcing a specific repo via env for reproduction + let repo_idx = std::env::var("FFF_FUZZ_REPO_IDX") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| ops.len() % REPO_POOL.len()); + let (repo_url, local_name) = REPO_POOL[repo_idx]; + eprintln!("=== fuzz_real_repos: repo={repo_url} ops={} ===", ops.len()); + + let scenario_start = Instant::now(); + let cached = ensure_repo_cloned(repo_url, local_name); + let tmp = tempfile::TempDir::new().unwrap(); + let workdir = tmp.path().join(local_name); + copy_repo_to_workdir(&cached, &workdir); + + // Ensure target/ is gitignored + let gitignore = workdir.join(".gitignore"); + let mut gi = fs::read_to_string(&gitignore).unwrap_or_default(); + if !gi.contains("target/") { + gi.push_str("\ntarget/\n"); + fs::write(&gitignore, &gi).unwrap(); + } + + let shared_picker = SharedFilePicker::default(); + FilePicker::new_with_shared_state( + shared_picker.clone(), + SharedFrecency::noop(), + FilePickerOptions { + base_path: workdir.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + watch: true, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .expect("FilePicker init"); + + let t0 = Instant::now(); + wait_for_bigram(&shared_picker); + let bigram_ms = t0.elapsed().as_secs_f64() * 1000.0; + + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let file_count = picker.get_files().len(); + eprintln!(" indexed {file_count} files, bigram ready in {bigram_ms:.0}ms"); + } + + // Advance mtime past scan timestamp + std::thread::sleep(Duration::from_millis(1100)); + + let mut tracked: Vec = Vec::new(); + let mut dead_markers: Vec = Vec::new(); + let mut ignored_markers: Vec = Vec::new(); + let mut ops_since_verify: usize = 0; + let mut text_files: Option> = None; + + for (op_idx, op) in ops.iter().enumerate() { + match op { + Op::CreateFile { seed } => { + let name = format!("fff_fuzz_new_{seed:08x}.rs"); + let marker = format!("FFF_FUZZ_NEW_{seed:08x}"); + // Marker appears only once on its own line + let content = format!("// {marker}\nfn placeholder() {{}}\n"); + fs::write(workdir.join(&name), content).unwrap(); + tracked.push(TrackedFile { + relative: name, + marker, + original_line: String::new(), + is_created: true, + last_write_sec: epoch_secs(), + }); + ops_since_verify += 1; + } + Op::EditTracked { seed } => { + if tracked.is_empty() { + continue; + } + let idx = *seed as usize % tracked.len(); + if tracked[idx].last_write_sec >= epoch_secs() { + std::thread::sleep(Duration::from_millis(1100)); + } + let new_marker = format!("FFF_FUZZ_EDIT_{seed:08x}"); + let path = workdir.join(&tracked[idx].relative); + // Replace the line containing our old marker with the new one + let old_marker_line = format!("// {}", tracked[idx].marker); + let content = fs::read_to_string(&path).unwrap_or_default(); + let new_content = content + .lines() + .map(|l| { + if l == old_marker_line { + format!("// {new_marker}") + } else { + l.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + fs::write(&path, new_content).unwrap(); + + dead_markers.push(tracked[idx].marker.clone()); + tracked[idx].marker = new_marker; + tracked[idx].last_write_sec = epoch_secs(); + ops_since_verify += 1; + } + Op::EditRandom { seed } => { + let files = text_files.get_or_insert_with(|| collect_text_files(&workdir)); + if files.is_empty() { + continue; + } + let target = &files[*seed as usize % files.len()]; + let relative = target + .strip_prefix(&workdir) + .unwrap() + .to_string_lossy() + .to_string(); + + if let Some(t) = tracked.iter().find(|t| t.relative == relative) { + if t.last_write_sec >= epoch_secs() { + std::thread::sleep(Duration::from_millis(1100)); + } + } + + let marker = format!("FFF_FUZZ_RAND_{seed:08x}"); + + // If already tracked, replace old marker line + if let Some(pos) = tracked.iter().position(|t| t.relative == relative) { + let old_marker_line = format!("// {}", tracked[pos].marker); + let content = fs::read_to_string(target).unwrap_or_default(); + let new_content = content + .lines() + .map(|l| { + if l == old_marker_line { + format!("// {marker}") + } else { + l.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + fs::write(target, new_content).unwrap(); + dead_markers.push(tracked[pos].marker.clone()); + tracked[pos].marker = marker; + tracked[pos].last_write_sec = epoch_secs(); + } else { + // First edit: inject marker at a deterministic line + let original = inject_marker(target, &marker, *seed).unwrap_or_default(); + tracked.push(TrackedFile { + relative, + marker, + original_line: original, + is_created: false, + last_write_sec: epoch_secs(), + }); + } + ops_since_verify += 1; + } + Op::DeleteTracked => { + if tracked.is_empty() { + continue; + } + let removed = tracked.swap_remove(0); + let abs = workdir.join(&removed.relative); + if abs.exists() { + if removed.is_created { + fs::remove_file(&abs).ok(); + } else { + let _ = Command::new("git") + .args(["rm", "-f", &removed.relative]) + .current_dir(&workdir) + .output(); + } + } + dead_markers.push(removed.marker); + text_files = None; // invalidate cache after deletion + ops_since_verify += 1; + } + Op::RevertTracked => { + // Revert a non-created tracked file using `git checkout` + // (restores original content, marker disappears) + let revertable = tracked.iter().position(|t| !t.is_created); + let Some(idx) = revertable else { continue }; + + if tracked[idx].last_write_sec >= epoch_secs() { + std::thread::sleep(Duration::from_millis(1100)); + } + + let _ = Command::new("git") + .args(["checkout", "--", &tracked[idx].relative]) + .current_dir(&workdir) + .output(); + + let reverted = tracked.swap_remove(idx); + dead_markers.push(reverted.marker); + text_files = None; // invalidate cache after revert + ops_since_verify += 1; + } + Op::IgnoredBurst { count, seed } => { + let dir = workdir.join("target/debug/build"); + fs::create_dir_all(&dir).unwrap(); + for i in 0..*count { + let marker = format!("FFF_IGN_{seed:08x}_{i}"); + fs::write( + dir.join(format!("ign_{seed:08x}_{i}.rs")), + format!("// {marker}\nfn {marker}() {{}}\n"), + ) + .unwrap(); + ignored_markers.push(marker); + } + ops_since_verify += 1; + } + Op::Verify => { + if tracked.is_empty() && dead_markers.is_empty() { + continue; + } + + // Poll until the watcher has propagated all pending events: + // all live markers findable, all dead markers gone, no ignored leaks. + let modes = [ + (GrepMode::PlainText, "Plain"), + (GrepMode::Regex, "Regex"), + (GrepMode::Fuzzy, "Fuzzy"), + ]; + let (mode, mode_name) = modes[op_idx % modes.len()]; + + let deadline = Instant::now() + CONVERGE_TIMEOUT; + let mut last_failure: Option = None; + + loop { + std::thread::sleep(WATCHER_SETTLE); + + // Write trigger to force a watcher batch + let trigger = workdir.join("fff_fuzz_trigger.rs"); + let _ = fs::write(&trigger, format!("// trigger {}\n", op_idx)); + + std::thread::sleep(WATCHER_SETTLE); + + let mut all_ok = true; + + // Check live markers (drop lock between each grep) + for tf in &tracked { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let found = grep_finds(picker, &tf.marker, mode); + drop(guard); + if !found { + last_failure = Some(format!( + "{mode_name} grep for {:?} in {:?} not found\n\ + is_created={} exists={} on_disk_has_marker={}", + tf.marker, + tf.relative, + tf.is_created, + workdir.join(&tf.relative).exists(), + fs::read_to_string(workdir.join(&tf.relative)) + .map(|c| c.contains(&tf.marker)) + .unwrap_or(false), + )); + all_ok = false; + break; + } + } + + // Check dead markers (only sample a few per iteration to + // avoid holding the lock too long with many dead markers) + if all_ok { + let sample_size = dead_markers.len().min(20); + for dead in dead_markers.iter().take(sample_size) { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let found = grep_finds(picker, dead, GrepMode::PlainText); + drop(guard); + if found { + last_failure = Some(format!("dead marker {dead:?} still findable")); + all_ok = false; + break; + } + } + } + + // Check ignored markers (sample first 5) + if all_ok { + for ig in ignored_markers.iter().take(5) { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let files = grep_file_list(picker, ig, GrepMode::PlainText); + drop(guard); + if !files.is_empty() { + last_failure = + Some(format!("ignored marker {ig:?} found in {files:?}")); + all_ok = false; + break; + } + } + } + + if all_ok { + eprintln!( + " op[{op_idx}] verify OK: {mode_name} mode, {} live, {} dead, {} ignored", + tracked.len(), + dead_markers.len(), + ignored_markers.len(), + ); + break; + } + + if Instant::now() >= deadline { + panic!( + "op[{op_idx}] verify TIMEOUT after {CONVERGE_TIMEOUT:?}:\n {}\n ops_since_last_verify={}", + last_failure.unwrap_or_default(), + ops_since_verify, + ); + } + } + + ops_since_verify = 0; + } + } + } + + // Final convergence: poll until everything is consistent + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + std::thread::sleep(WATCHER_SETTLE); + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let live_ok = tracked + .iter() + .all(|tf| grep_finds(picker, &tf.marker, GrepMode::PlainText)); + let dead_ok = dead_markers + .iter() + .all(|d| !grep_finds(picker, d, GrepMode::PlainText)); + drop(guard); + + if live_ok && dead_ok { + break; + } + assert!( + Instant::now() < deadline, + "final verify TIMEOUT: live_ok={live_ok} dead_ok={dead_ok}" + ); + } + + // Teardown + shared_picker.wait_for_indexing_complete(Duration::from_secs(30)); + if let Ok(mut guard) = shared_picker.write() { + if let Some(mut picker) = guard.take() { + picker.stop_background_monitor(); + } + } + + eprintln!( + " PASSED: {} ops, {} tracked, {} dead, {} ignored ({:.1}s)", + ops.len(), + tracked.len(), + dead_markers.len(), + ignored_markers.len(), + scenario_start.elapsed().as_secs_f64(), + ); +} + +// ================ +// Proptest harness +// ================= +// +fn proptest_config() -> ProptestConfig { + ProptestConfig { + cases: fuzz_cases(), + max_shrink_iters: 0, + fork: false, + failure_persistence: Some(Box::new(FileFailurePersistence::Direct(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fuzz_real_repos.proptest-regressions", + )))), + ..ProptestConfig::default() + } +} + +#[derive(Debug, Clone)] +enum Op { + CreateFile { seed: u32 }, + EditTracked { seed: u32 }, + EditRandom { seed: u32 }, + DeleteTracked, + RevertTracked, + IgnoredBurst { count: u8, seed: u32 }, + Verify, +} + +fn op_strategy() -> impl Strategy { + prop_oneof![ + 12 => any::().prop_map(|s| Op::CreateFile { seed: s }), + 18 => any::().prop_map(|s| Op::EditTracked { seed: s }), + 18 => any::().prop_map(|s| Op::EditRandom { seed: s }), + 8 => Just(Op::DeleteTracked), + 10 => Just(Op::RevertTracked), + 9 => (1u8..20, any::()).prop_map(|(c, s)| Op::IgnoredBurst { count: c, seed: s }), + // Explicit verification rounds + 25 => Just(Op::Verify), + ] +} + +fn ops_strategy() -> impl Strategy> { + let min = fuzz_min_ops(); + let max = fuzz_max_ops(); + prop::collection::vec(op_strategy(), min..=max) +} + +proptest! { + #![proptest_config(proptest_config())] + + #[test] + fn fuzz_real_repos_proptest(ops in ops_strategy()) { + run_scenario(&ops); + } +} diff --git a/crates/fff-core/tests/grep_integration.rs b/crates/fff-core/tests/grep_integration.rs new file mode 100644 index 0000000..2765de3 --- /dev/null +++ b/crates/fff-core/tests/grep_integration.rs @@ -0,0 +1,1837 @@ +use std::fs; +use std::path::Path; +use tempfile::TempDir; + +use fff_search::FilePickerOptions; +use fff_search::file_picker::FilePicker; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; + +/// Build a batch of test files and return a FilePicker with them indexed. +fn create_picker(base: &Path, specs: &[(&str, &str)]) -> FilePicker { + for (rel, contents) in specs { + let full_path = base.join(rel); + if let Some(parent) = full_path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(&full_path, contents).unwrap(); + } + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + watch: false, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + picker +} + +/// Shorthand to build default options for plain text mode. +fn plain_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +/// Shorthand to build default options for regex mode. +fn regex_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::Regex, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +/// Shorthand to build default options for fuzzy mode. +fn fuzzy_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::Fuzzy, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +#[test] +fn plain_text_finds_exact_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("hello.txt", "Hello, World!\nGoodbye, World!\n")], + ); + + let parsed = parse_grep_query("Hello"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); + assert!(result.matches[0].line_content.contains("Hello")); +} + +#[test] +fn plain_text_smart_case_insensitive() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "Hello World\nhello world\nHELLO WORLD\n")], + ); + + // All lowercase query → smart case → case-insensitive + let parsed = parse_grep_query("hello"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 3, + "smart case should match all 3 lines" + ); +} + +#[test] +fn plain_text_smart_case_sensitive_with_uppercase() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "Hello World\nhello world\nHELLO WORLD\n")], + ); + + // Query has uppercase → smart case → case-sensitive + let parsed = parse_grep_query("Hello"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "uppercase in query should trigger case-sensitive" + ); + assert_eq!(result.matches[0].line_number, 1); +} + +#[test] +fn plain_text_regex_metacharacters_are_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "code.rs", + "fn main() {\n println!(\"test\");\n}\nfn foo() {}\n", + )], + ); + + // In plain text mode, these regex metacharacters should be literal + let parsed = parse_grep_query("fn main()"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); + + // Parentheses should NOT be treated as regex groups + let parsed2 = parse_grep_query("(\"test\")"); + let result2 = picker.grep(&parsed2, &plain_opts()); + assert_eq!(result2.matches.len(), 1); + assert_eq!(result2.matches[0].line_number, 2); +} + +#[test] +fn plain_text_dot_is_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "config.toml", + "version = \"1.0\"\nname = \"foo\"\nversion_major = 1\n", + )], + ); + + // In plain text mode, dot should be literal, not "any char" + let parsed = parse_grep_query("1.0"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "dot should be literal, not 'any char'" + ); + assert!(result.matches[0].line_content.contains("1.0")); +} + +#[test] +fn plain_text_asterisk_is_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "doc.md", + "Use **bold** text\nUse *italic* text\nUse normal text\n", + )], + ); + + let parsed = parse_grep_query("**bold**"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); +} + +#[test] +fn plain_text_backslash_is_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("paths.txt", "C:\\Users\\foo\\bar\n/home/user/bin\n")], + ); + + let parsed = parse_grep_query("C:\\Users"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!(result.matches.len(), 1); +} + +#[test] +fn plain_text_across_multiple_files() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("a.txt", "use std::io;\nuse std::fs;\n"), + ("b.txt", "use std::path;\nuse serde;\n"), + ("c.txt", "no match here\n"), + ], + ); + + let parsed = parse_grep_query("use std"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 3); + // Should match in files a.txt and b.txt + assert_eq!(result.files.len(), 2); +} + +#[test] +fn plain_text_highlight_offsets_are_correct() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "foo bar foo baz foo\n")]); + + let parsed = parse_grep_query("foo"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + // "foo" appears at byte offsets 0, 8, 16 + assert_eq!(m.match_byte_offsets.len(), 3); + assert_eq!(m.match_byte_offsets[0], (0, 3)); + assert_eq!(m.match_byte_offsets[1], (8, 11)); + assert_eq!(m.match_byte_offsets[2], (16, 19)); + assert_eq!(m.col, 0); +} + +#[test] +fn plain_text_empty_query_returns_no_content_matches() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "some content\n")]); + + let parsed = parse_grep_query(""); + let result = picker.grep(&parsed, &plain_opts()); + + // Empty query in grep returns git-modified welcome state (no content matches) + // Since our test files have no git status, we expect 0 matches + assert_eq!(result.matches.len(), 0); +} + +#[test] +fn plain_text_binary_files_are_skipped() { + let tmp = TempDir::new().unwrap(); + let mut bin_content = b"match this text\n".to_vec(); + bin_content.extend_from_slice(&[0u8; 100]); // NUL bytes make it binary + bin_content.extend_from_slice(b"match this text\n"); + + // Initialize a git repo so the picker indexes binary-extension files + // (marking them as binary) rather than skipping them entirely. + std::process::Command::new("git") + .args(["init", "--quiet"]) + .current_dir(tmp.path()) + .status() + .expect("git init failed"); + + // Use a known binary extension so the picker's scan-time heuristic marks + // it as binary without needing mutable post-hoc access. + fs::write(tmp.path().join("binary.png"), &bin_content).unwrap(); + + // create_picker writes text.txt and then scans the directory, picking up + // both binary.png (already on disk) and text.txt. + let picker = create_picker(tmp.path(), &[("text.txt", "match this text\n")]); + + // binary.png should have been auto-detected as binary by extension heuristic + let has_binary = picker + .get_files() + .iter() + .any(|f| f.relative_path(&picker).contains("binary.png") && f.is_binary()); + assert!(has_binary, "binary.png should be detected as binary"); + + let parsed = parse_grep_query("match this text"); + let result = picker.grep(&parsed, &plain_opts()); + + // Only the text file should be searched, not the binary one + assert_eq!(result.files.len(), 1); + assert!(result.files[0].relative_path(&picker).contains("text.txt")); +} + +#[test] +fn binary_payload_after_long_ascii_header_is_detected() { + // Mimics formats like Radiance .hdr / Apple bplist / Adobe .ai where the + // first ~1KB is plain ASCII and the binary payload (NULs) starts later. + // The legacy 512-byte sniff missed these; the bigram-build memchr scan + // over the whole indexed buffer must catch them. + use fff_search::file_picker::FFFMode; + use fff_search::{SharedFilePicker, SharedFrecency}; + use std::time::Duration; + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + let mut content = Vec::new(); + // 1 KiB of plain ASCII header — escapes any small fixed-window NUL sniff. + content.extend(std::iter::repeat_n(b'A', 1024)); + content.extend_from_slice(b"\nmatch this text\n"); + // Binary payload: NUL bytes that prove the file is not text. + content.extend(std::iter::repeat_n(0u8, 256)); + content.extend_from_slice(b"\nmatch this text\n"); + + // Use a *text* extension so the scan-time heuristic does NOT pre-flag it. + // Only the bigram-time content scan can mark it binary. + fs::write(base.join("header.txt"), &content).unwrap(); + fs::write(base.join("plain.txt"), b"match this text\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(25)); + let ready = shared_picker + .read() + .ok() + .and_then(|g| { + g.as_ref() + .map(|p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let was_flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("header.txt") && f.is_binary()); + assert!( + was_flagged, + "header.txt with NULs past 512 bytes must be flagged binary by the whole-buffer memchr scan" + ); + + let parsed = parse_grep_query("match this text"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!( + result.files.len(), + 1, + "only plain.txt should be searched; header.txt must be skipped as binary" + ); + assert!( + result.files[0].relative_path(picker).contains("plain.txt"), + "the only match should come from plain.txt" + ); +} + +#[test] +fn unknown_extension_binary_added_after_scan_is_reclassified() { + // The initial-scan path runs detect_binary_content as part of bigram build, + // but the watcher path used to fall back to extension-only triage and + // missed binary files with unknown extensions like `.codex`. + use fff_search::file_picker::FFFMode; + use fff_search::{SharedFilePicker, SharedFrecency}; + use std::time::Duration; + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // Seed one tracked text file so the initial scan has something to work with. + fs::write(base.join("seed.txt"), b"seed\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(25)); + let ready = shared_picker + .read() + .ok() + .and_then(|g| { + g.as_ref() + .map(|p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } + + // Drop the file on disk after indexing finished, then announce it through + // the watcher entry point. `.codex` is intentionally not in the extension + // allow-list — only a content sniff can flag it. + let mut payload = vec![0x03u8, 0x00, 0x04, 0x05]; + payload.extend(std::iter::repeat_n(0u8, 256)); + let new_path = base.join("snapshot.codex"); + fs::write(&new_path, &payload).unwrap(); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker.handle_create_or_modify(&new_path).is_some(), + "handle_create_or_modify must accept the new file" + ); + } + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let was_flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("snapshot.codex") && f.is_binary()); + assert!( + was_flagged, + "snapshot.codex must be flagged binary when added via the watcher path" + ); +} + +#[test] +fn text_file_modified_to_binary_is_reclassified() { + // A file that started life as text and later got rewritten with NUL bytes + // (e.g. a generator overwrote a .log) must lose its text classification. + use fff_search::file_picker::FFFMode; + use fff_search::{SharedFilePicker, SharedFrecency}; + use std::time::Duration; + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // Start as plain text with a known extension. + fs::write(base.join("notes.txt"), b"hello world\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(25)); + let ready = shared_picker + .read() + .ok() + .and_then(|g| { + g.as_ref() + .map(|p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } + + // Sanity: it's text right now. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let is_text = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("notes.txt") && !f.is_binary()); + assert!(is_text, "notes.txt should start as text"); + } + + // Overwrite with binary content and replay through the watcher entry point. + // Bump mtime so update_metadata records it as a real change. + std::thread::sleep(Duration::from_secs(1)); + let mut payload = b"header text\n".to_vec(); + payload.extend(std::iter::repeat_n(0u8, 256)); + fs::write(base.join("notes.txt"), &payload).unwrap(); + + { + let mut guard = shared_picker.write().unwrap(); + let picker = guard.as_mut().unwrap(); + assert!( + picker + .handle_create_or_modify(base.join("notes.txt")) + .is_some(), + "handle_create_or_modify must succeed for the modify case" + ); + } + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let now_binary = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("notes.txt") && f.is_binary()); + assert!( + now_binary, + "notes.txt must flip to binary after being overwritten with NULs" + ); +} + +#[test] +fn large_unknown_extension_binary_is_classified_at_scan_time() { + // Files larger than MAX_INDEXABLE_FILE_SIZE never enter build_bigram_index, + // so without a separate header sniff they default to is_binary=false and + // pollute grep results with NUL-laden lines (e.g. a committed ELF blob + // named `codex_view` with no extension). + use fff_search::file_picker::FFFMode; + use fff_search::grep::{GrepSearchOptions, parse_grep_query}; + use fff_search::{SharedFilePicker, SharedFrecency}; + use std::time::Duration; + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // 3 MiB: above the 2 MiB bigram cap and below the 10 MiB grep cap. + // ELF-like header with NULs at the very start, then ASCII filler so a + // grep for "match this text" would otherwise return polluted lines. + let mut blob = Vec::new(); + blob.extend_from_slice(b"\x7fELF\x02\x01\x01\x00"); + blob.extend(std::iter::repeat_n(0u8, 256)); + blob.extend_from_slice(b"\nmatch this text\n"); + blob.extend(std::iter::repeat_n(b'A', 3 * 1024 * 1024)); + blob.extend_from_slice(b"\nmatch this text\n"); + fs::write(base.join("codex_view"), &blob).unwrap(); + fs::write(base.join("plain.txt"), b"match this text\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let deadline = std::time::Instant::now() + Duration::from_secs(15); + loop { + std::thread::sleep(Duration::from_millis(25)); + let ready = shared_picker + .read() + .ok() + .and_then(|g| { + g.as_ref() + .map(|p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let was_flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("codex_view") && f.is_binary()); + assert!( + was_flagged, + "large no-extension binary must be flagged via the header sniff" + ); + + let parsed = parse_grep_query("match this text"); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + ..plain_opts() + }; + let result = picker.grep(&parsed, &opts); + assert_eq!( + result.files.len(), + 1, + "only plain.txt should be searched; codex_view must be skipped as binary" + ); + assert!( + result.files[0].relative_path(picker).contains("plain.txt"), + "the only match should come from plain.txt" + ); +} + +#[test] +fn large_binary_with_nuls_past_header_is_classified() { + // Guards the streaming sniff: a >2 MB file that is pure ASCII well past any + // fixed header window (the old code only checked the first 8 KB) but has + // NULs deeper in. Grep reads the whole file up to max_file_size, so the + // detector must scan the same range or the binary tail leaks as "text". + use fff_search::file_picker::FFFMode; + use fff_search::grep::{GrepSearchOptions, parse_grep_query}; + use fff_search::{SharedFilePicker, SharedFrecency}; + use std::time::Duration; + + let tmp = TempDir::new().unwrap(); + let base = tmp.path(); + + // 1 MiB of clean ASCII (with a grep marker) — dwarfs any header sniff — + // then NUL bytes, keeping the total above the 2 MiB non-indexable cap. + let mut blob = Vec::new(); + blob.extend_from_slice(b"match this text\n"); + blob.extend(std::iter::repeat_n(b'A', 1024 * 1024)); + blob.extend_from_slice(b"match this text\n"); + blob.extend(std::iter::repeat_n(0u8, 1024 * 1024 + 4096)); // NULs start ~1 MiB in + blob.extend_from_slice(b"match this text\n"); + assert!(blob.len() > 2 * 1024 * 1024); + fs::write(base.join("late_nul.dat"), &blob).unwrap(); + fs::write(base.join("plain.txt"), b"match this text\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + let deadline = std::time::Instant::now() + Duration::from_secs(15); + loop { + std::thread::sleep(Duration::from_millis(25)); + let ready = shared_picker + .read() + .ok() + .and_then(|g| { + g.as_ref() + .map(|p| !p.is_scan_active() && p.bigram_index().is_some()) + }) + .unwrap_or(false); + if ready { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Timed out waiting for bigram build" + ); + } + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("late_nul.dat") && f.is_binary()); + assert!( + flagged, + "NULs past the 8 KB header window must still be detected by the streaming scan" + ); + + let parsed = parse_grep_query("match this text"); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + ..plain_opts() + }; + let result = picker.grep(&parsed, &opts); + assert_eq!( + result.files.len(), + 1, + "only plain.txt should match; late_nul.dat must be skipped as binary" + ); + assert!(result.files[0].relative_path(picker).contains("plain.txt")); +} + +#[test] +fn plain_text_max_matches_per_file() { + let tmp = TempDir::new().unwrap(); + let mut content = String::new(); + for i in 0..50 { + content.push_str(&format!("line {} match_target\n", i)); + } + fs::write(tmp.path().join("many.txt"), &content).unwrap(); + let picker = create_picker(tmp.path(), &[("many.txt", &content)]); + + let mut opts = plain_opts(); + opts.max_matches_per_file = 5; + + let parsed = parse_grep_query("match_target"); + let result = picker.grep(&parsed, &opts); + + assert_eq!( + result.matches.len(), + 5, + "should cap at max_matches_per_file" + ); +} + +#[test] +fn plain_text_page_limit() { + let tmp = TempDir::new().unwrap(); + let mut content = String::new(); + for i in 0..100 { + content.push_str(&format!("line {} target\n", i)); + } + fs::write(tmp.path().join("big.txt"), &content).unwrap(); + let picker = create_picker(tmp.path(), &[("big.txt", &content)]); + + let mut opts = plain_opts(); + opts.page_limit = 10; + + let parsed = parse_grep_query("target"); + let result = picker.grep(&parsed, &opts); + + // page_limit is a soft minimum: we always finish the current file, so we + // get at least page_limit matches (no data loss) and at most + // max_matches_per_file (200) from a single file. + assert!( + result.matches.len() >= opts.page_limit, + "should return at least page_limit matches: got {}", + result.matches.len() + ); + assert!( + result.matches.len() <= opts.max_matches_per_file, + "should never exceed max_matches_per_file: got {}", + result.matches.len() + ); + // Single file with 100 lines all matching — all should be returned. + assert_eq!(result.matches.len(), 100, "all 100 lines must be returned"); +} + +#[test] +fn plain_text_file_offset_pagination() { + let tmp = TempDir::new().unwrap(); + // Create many files (1 match per file) so file-based pagination exercises + // offset tracking across files with and without matches. + let specs: Vec<(String, String)> = (0..20) + .map(|i| { + ( + format!("file_{:02}.txt", i), + format!("unique_token_{}\n", i), + ) + }) + .collect(); + let spec_refs: Vec<(&str, &str)> = specs + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(); + let picker = create_picker(tmp.path(), &spec_refs); + + let mut opts = plain_opts(); + opts.page_limit = 5; + + // Collect ALL matches across all pages and verify no duplicates and full coverage. + let mut all_line_texts: Vec = Vec::new(); + let mut pages = 0; + let max_pages = 20; // safety limit + + loop { + let parsed = parse_grep_query("unique_token"); + let result = picker.grep(&parsed, &opts); + + for m in &result.matches { + let text = m.line_content.trim().to_string(); + assert!( + !all_line_texts.contains(&text), + "duplicate match across pages: '{}'", + text + ); + all_line_texts.push(text); + } + + pages += 1; + assert!(pages <= max_pages, "pagination did not terminate"); + + if result.next_file_offset == 0 { + break; + } + + // Offset must strictly advance + assert!( + result.next_file_offset > opts.file_offset, + "next_file_offset ({}) did not advance past current ({})", + result.next_file_offset, + opts.file_offset + ); + opts.file_offset = result.next_file_offset; + } + + assert_eq!( + all_line_texts.len(), + 20, + "pagination should find all 20 matches across all pages, got {}", + all_line_texts.len() + ); + assert!( + pages > 1, + "should require multiple pages with page_limit=5 and 20 files" + ); +} + +#[test] +fn plain_text_line_numbers_are_correct() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "line one\nline two\nline three\nline four\n")], + ); + + let parsed = parse_grep_query("line"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 4); + assert_eq!(result.matches[0].line_number, 1); + assert_eq!(result.matches[1].line_number, 2); + assert_eq!(result.matches[2].line_number, 3); + assert_eq!(result.matches[3].line_number, 4); +} + +#[test] +fn plain_text_max_file_size_filter() { + let tmp = TempDir::new().unwrap(); + // Create a file larger than 100 bytes + let big_content = "a".repeat(200) + "\nmatch_me\n"; + fs::write(tmp.path().join("big.txt"), &big_content).unwrap(); + let picker = create_picker(tmp.path(), &[("big.txt", &big_content)]); + + let mut opts = plain_opts(); + opts.max_file_size = 100; // Only allow files up to 100 bytes + + let parsed = parse_grep_query("match_me"); + let result = picker.grep(&parsed, &opts); + + assert_eq!(result.matches.len(), 0, "large file should be filtered out"); + assert_eq!(result.filtered_file_count, 0); +} + +// ── Regex mode tests ─────────────────────────────────────────────────── + +#[test] +fn regex_basic_pattern() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "foo123\nbar456\nbaz789\nfoo_bar\n")], + ); + + let parsed = parse_grep_query("foo\\d+"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); + assert!(result.matches[0].line_content.contains("foo123")); +} + +#[test] +fn regex_capture_group_matching() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "foobar\nfoobaz\nfoo123\n")]); + + // Use a capturing group (not lookahead, which regex crate doesn't support) + let parsed = parse_grep_query("foo(bar|baz)"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 2); + let contents: Vec<&str> = result + .matches + .iter() + .map(|m| m.line_content.as_str()) + .collect(); + assert!(contents.contains(&"foobar")); + assert!(contents.contains(&"foobaz")); +} + +#[test] +fn regex_dot_matches_any_char() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "v1.0\nv1x0\nv1-0\nv100\nv2.0\n")]); + + // In regex mode, . matches any character, so v1.0 matches v1.0, v1x0, v1-0, and v100 + let parsed = parse_grep_query("v1.0"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!( + result.matches.len(), + 4, + "regex dot should match v1.0, v1x0, v1-0, and v100 (dot matches any char)" + ); +} + +#[test] +fn regex_alternation() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "apple\nbanana\ncherry\napricot\n")]); + + let parsed = parse_grep_query("apple|cherry"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 2); + let lines: Vec = result.matches.iter().map(|m| m.line_number).collect(); + assert!(lines.contains(&1)); + assert!(lines.contains(&3)); +} + +#[test] +fn regex_character_class() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "cat\ncut\ncot\ncit\ncxt\n")]); + + let parsed = parse_grep_query("c[aou]t"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 3); + let contents: Vec<&str> = result + .matches + .iter() + .map(|m| m.line_content.as_str()) + .collect(); + assert!(contents.contains(&"cat")); + assert!(contents.contains(&"cut")); + assert!(contents.contains(&"cot")); +} + +#[test] +fn regex_quantifiers() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "fo\nfoo\nfooo\nfoooo\nbar\n")]); + + let parsed = parse_grep_query("fo{2,3}"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 3, "should match foo, fooo, foooo"); +} + +#[test] +fn regex_anchors() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "start of line\nmiddle start end\nend of line\n")], + ); + + let parsed = parse_grep_query("^start"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); +} + +#[test] +fn regex_anchors_multiword() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "test.c", + "int ff_function(void);\nstatic int ff_other(void);\nint main(void);\nint ff_another(void);\n", + )], + ); + + // ^int ff_ should match lines starting with "int ff_" + let parsed = parse_grep_query("^int ff_"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!( + result.matches.len(), + 2, + "should match 2 lines starting with 'int ff_'" + ); + assert!(result.matches[0].line_content.contains("ff_function")); + assert!(result.matches[1].line_content.contains("ff_another")); +} + +#[test] +fn regex_highlight_offsets_variable_length() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "aab aaab aaaab\n")]); + + let parsed = parse_grep_query("a+b"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + // Regex "a+b" matches "aab" (3 bytes), "aaab" (4 bytes), "aaaab" (5 bytes) + assert_eq!(m.match_byte_offsets.len(), 3); + // Verify the match spans have different lengths (variable-length regex) + assert_eq!(m.match_byte_offsets[0], (0, 3)); // "aab" + assert_eq!(m.match_byte_offsets[1], (4, 8)); // "aaab" + assert_eq!(m.match_byte_offsets[2], (9, 14)); // "aaaab" +} + +#[test] +fn regex_invalid_pattern_falls_back_to_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "call name(arg)\nother line\n")]); + + // Invalid regex: unmatched group — should fall back to literal search + let parsed = parse_grep_query("name("); + let result = picker.grep(&parsed, ®ex_opts()); + + // Fallback to literal: finds "name(" in "call name(arg)" + assert_eq!( + result.matches.len(), + 1, + "invalid regex should fall back to literal and find the match" + ); + assert!( + result.regex_fallback_error.is_some(), + "should report the regex compilation error" + ); + assert!(result.matches[0].line_content.contains("name(")); + + // A pattern that doesn't exist anywhere — still falls back but finds nothing + let parsed2 = parse_grep_query("zzz("); + let result2 = picker.grep(&parsed2, ®ex_opts()); + assert_eq!(result2.matches.len(), 0); + assert!(result2.regex_fallback_error.is_some()); +} + +#[test] +fn regex_smart_case() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "Foo bar\nfoo bar\nFOO BAR\n")]); + + // Lowercase query → case-insensitive + let parsed_lower = parse_grep_query("foo"); + let result_lower = picker.grep(&parsed_lower, ®ex_opts()); + assert_eq!(result_lower.matches.len(), 3); + + // Query with uppercase → case-sensitive + let parsed_upper = parse_grep_query("Foo"); + let result_upper = picker.grep(&parsed_upper, ®ex_opts()); + assert_eq!(result_upper.matches.len(), 1); +} + +#[test] +fn regex_across_multiple_files() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("lib.rs", "fn main() {}\nfn helper() {}\nstruct Foo;\n"), + ( + "test.rs", + "fn test_one() {}\nfn test_two() {}\nmod tests;\n", + ), + ("readme.md", "# Title\nSome text\n"), + ], + ); + + let parsed = parse_grep_query("fn \\w+\\(\\)"); + let result = picker.grep(&parsed, ®ex_opts()); + + // Should match: fn main(), fn helper(), fn test_one(), fn test_two() + assert_eq!(result.matches.len(), 4); + assert_eq!(result.files.len(), 2, "matches in 2 .rs files, not readme"); +} + +// ── Mode comparison tests ────────────────────────────────────────────── + +#[test] +fn plain_text_and_regex_agree_on_simple_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "hello world\ngoodbye world\nhello again\n")], + ); + + let parsed = parse_grep_query("hello"); + let plain_result = picker.grep(&parsed, &plain_opts()); + let regex_result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!(plain_result.matches.len(), regex_result.matches.len()); + for (p, r) in plain_result.matches.iter().zip(regex_result.matches.iter()) { + assert_eq!(p.line_number, r.line_number); + assert_eq!(p.line_content, r.line_content); + } +} + +#[test] +fn plain_text_escapes_what_regex_does_not() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "price is $100\nprice is 100\nprice is $200\n")], + ); + + // "$100" — in plain text, $ is literal; in regex, $ is anchor + let parsed_plain = parse_grep_query("$100"); + let plain_result = picker.grep(&parsed_plain, &plain_opts()); + let parsed_regex = parse_grep_query("\\$100"); + let regex_result = picker.grep(&parsed_regex, ®ex_opts()); + + // Plain text should find "$100" literally + assert_eq!(plain_result.matches.len(), 1); + assert!(plain_result.matches[0].line_content.contains("$100")); + + // Regex with escaped $ should also find "$100" + assert_eq!(regex_result.matches.len(), 1); +} + +// ── Constraint integration tests ─────────────────────────────────────── + +#[test] +fn grep_with_extension_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("a.rs", "use std::io;\nfn main() {}\n"), + ("b.txt", "use std::io;\nsome text\n"), + ("c.rs", "use std::fs;\n"), + ], + ); + + let parsed = parse_grep_query("use std *.rs"); + let result = picker.grep(&parsed, &plain_opts()); + + // Should only search .rs files + for file in &result.files { + assert!( + file.relative_path(&picker).ends_with(".rs"), + "should only match .rs files, got: {}", + file.relative_path(&picker) + ); + } + assert!( + result.matches.len() >= 2, + "should find matches in .rs files" + ); +} + +// ── Bracket / glob character tests ───────────────────────────────────── + +#[test] +fn plain_text_bracket_is_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "code.rs", + "let x = arr[0];\nlet y = arr[1];\nlet z = something;\n", + )], + ); + + let parsed = parse_grep_query("arr[0]"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "brackets should be literal in plain text mode" + ); + assert_eq!(result.matches[0].line_number, 1); +} + +// ── Backslash escape tests ───────────────────────────────────────────── + +#[test] +fn grep_backslash_escapes_extension_filter() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("a.rs", "contains *.rs pattern\n"), + ("b.txt", "also has *.rs here\n"), + ], + ); + + // Without escape: "*.rs" is an extension filter, so only .rs files are searched + let parsed = parse_grep_query("pattern *.rs"); + let result_filter = picker.grep(&parsed, &plain_opts()); + assert_eq!( + result_filter.files.len(), + 1, + "*.rs should filter to .rs files" + ); + + // With escape: "\*.rs" is literal text, both files are searched + let parsed_escaped = parse_grep_query("\\*.rs"); + let result_literal = picker.grep(&parsed_escaped, &plain_opts()); + assert_eq!( + result_literal.matches.len(), + 2, + "\\*.rs should search for literal *.rs in all files" + ); +} + +#[test] +fn grep_backslash_escapes_path_segment() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("src/main.rs", "search for /src/ path\n"), + ("lib/utils.rs", "also /src/ mentioned\n"), + ], + ); + + // With escape: "\\/src/" is literal text, not a path constraint + let parsed = parse_grep_query("\\/src/"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!( + result.matches.len(), + 2, + "\\/src/ should search for literal /src/ in all files" + ); +} + +#[test] +fn grep_backslash_escapes_negation() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "the !test macro\nother stuff\n")]); + + // With escape: "\\!test" is literal text "!test" + let parsed = parse_grep_query("\\!test"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!(result.matches.len(), 1); + assert!(result.matches[0].line_content.contains("!test")); +} + +#[test] +fn grep_with_path_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("src/lib.rs", "target_text\n"), + ("tests/test.rs", "target_text\n"), + ("src/main.rs", "other content\n"), + ], + ); + + let parsed = parse_grep_query("target_text /src/"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + let rel = result.files[0].relative_path(&picker); + assert!(rel.starts_with("src/") || rel.starts_with("src\\")); +} + +// ── Negated constraint tests ─────────────────────────────────────────── + +#[test] +fn grep_with_negated_extension_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("src/lib.rs", "target_text\n"), + ("src/app.ts", "target_text\n"), + ("src/main.rs", "target_text\n"), + ], + ); + + let query = "target_text !*.rs"; + let parsed = parse_grep_query(query); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "should only find matches in non-.rs files, got {} matches", + result.matches.len() + ); + assert!( + result.files[0].relative_path(&picker).ends_with(".ts"), + "should only match .ts file, got: {}", + result.files[0].relative_path(&picker) + ); +} + +#[test] +fn grep_with_negated_path_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("src/lib.rs", "target_text\n"), + ("tests/test.rs", "target_text\n"), + ("src/main.rs", "other content\n"), + ], + ); + + let query = "target_text !/src/"; + let parsed = parse_grep_query(query); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "should only find matches outside src/, got {} matches", + result.matches.len() + ); + assert!( + result.files[0].relative_path(&picker).starts_with("tests/") + || result.files[0] + .relative_path(&picker) + .starts_with("tests\\"), + "should only match tests/ file, got: {}", + result.files[0].relative_path(&picker) + ); +} + +#[test] +fn grep_with_negated_text_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("src/lib.rs", "target_text\n"), + ("tests/helper.rs", "target_text\n"), + ("docs/readme.md", "target_text\n"), + ], + ); + + let query = "target_text !test"; + let parsed = parse_grep_query(query); + let result = picker.grep(&parsed, &plain_opts()); + + // "tests/helper.rs" contains "test" in path, should be excluded + assert_eq!( + result.matches.len(), + 2, + "should find matches in files without 'test' in path, got {} matches", + result.matches.len() + ); + for file in &result.files { + assert!( + !file.relative_path(&picker).contains("test"), + "should not match files with 'test' in path, got: {}", + file.relative_path(&picker) + ); + } +} + +// ── Edge case tests ──────────────────────────────────────────────────── + +#[test] +fn grep_empty_file_is_skipped() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("empty.txt", ""), ("text.txt", "findme\n")]); + + let parsed = parse_grep_query("findme"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); +} + +#[test] +fn grep_single_line_no_trailing_newline() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "no newline at end")]); + + let parsed = parse_grep_query("no newline"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 1); +} + +#[test] +fn grep_unicode_content() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("utf8.txt", "日本語テスト\nrégulière\nñoño\n")], + ); + + let parsed = parse_grep_query("régulière"); + let result = picker.grep(&parsed, &plain_opts()); + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].line_number, 2); + + let parsed2 = parse_grep_query("ñoño"); + let result2 = picker.grep(&parsed2, &plain_opts()); + assert_eq!(result2.matches.len(), 1); + assert_eq!(result2.matches[0].line_number, 3); +} + +#[test] +fn grep_long_line_is_truncated() { + let tmp = TempDir::new().unwrap(); + let long_line = format!("{}NEEDLE{}", "x".repeat(1000), "y".repeat(1000)); + fs::write(tmp.path().join("long.txt"), &long_line).unwrap(); + let picker = create_picker(tmp.path(), &[("long.txt", &long_line)]); + + let parsed = parse_grep_query("NEEDLE"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + // The line_content should be truncated to MAX_LINE_DISPLAY_LEN (512) + assert!( + result.matches[0].line_content.len() <= 512, + "line should be truncated: len={}", + result.matches[0].line_content.len() + ); +} + +#[test] +fn regex_word_boundary() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "foo\nfoobar\nbarfoo\nfoo_baz\n")]); + + let parsed = parse_grep_query("\\bfoo\\b"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!( + result.matches.len(), + 1, + "only exact word 'foo' should match" + ); + assert_eq!(result.matches[0].line_number, 1); +} + +#[test] +fn plain_text_question_mark_is_literal() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "a.txt", + "what is this?\nhow does it work?\nno question here\nwhat?\n", + )], + ); + + let parsed = parse_grep_query("?"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 3, + "question mark should be literal in plain text mode" + ); +} + +#[test] +fn plain_text_query_with_question_mark_in_word() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "code.rs", + "let x = foo?;\nlet y = bar.baz();\nfoo?.unwrap()\n", + )], + ); + + let parsed = parse_grep_query("foo?"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 2, + "should find 'foo?' literally in both lines" + ); +} + +#[test] +fn regex_question_mark_is_quantifier() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "color\ncolour\ncolouur\n")]); + + // In regex mode, ? means "zero or one of preceding" + let parsed = parse_grep_query("colou?r"); + let result = picker.grep(&parsed, ®ex_opts()); + + assert_eq!( + result.matches.len(), + 2, + "regex ? should match 'color' and 'colour' but not 'colouur'" + ); +} + +// ── Fuzzy mode tests ─────────────────────────────────────────────────── + +#[test] +fn fuzzy_finds_exact_substring() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("a.txt", "hello world\ngoodbye world\nhello again\n")], + ); + + let parsed = parse_grep_query("hello"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + assert_eq!( + result.matches.len(), + 2, + "fuzzy should find 'hello' in both lines" + ); + assert!(result.matches[0].line_content.contains("hello")); + assert!(result.matches[1].line_content.contains("hello")); +} + +#[test] +fn fuzzy_finds_scattered_characters() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "code.rs", + "fn mutex_lock() {}\nfn main() {}\nfn mutex_unlock() {}\n", + )], + ); + + // "mutex" should fuzzy match "mutex_lock" (contiguous prefix) + let parsed = parse_grep_query("mutex"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + assert!( + !result.matches.is_empty(), + "fuzzy should find 'mutex' in 'mutex_lock'" + ); + assert!(result.matches[0].line_content.contains("mutex_lock")); +} + +#[test] +fn fuzzy_highlight_offsets_correct() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "hello world\n")]); + + let parsed = parse_grep_query("hell"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + + // "hell" should match 'h'(0), 'e'(1), 'l'(2), 'l'(3) in "hello" + // These should be converted to byte offsets + assert!( + !m.match_byte_offsets.is_empty(), + "should have highlight offsets" + ); +} + +#[test] +fn fuzzy_unicode_char_indices() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("utf8.txt", "日本語テスト\nrégulière\nñoño\n")], + ); + + // Use "guli" which is a contiguous ASCII substring within "régulière" + // (the chars g-u-l-i appear contiguously between the two accented chars) + let parsed = parse_grep_query("guli"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + // Should fuzzy match "régulière" (with multi-byte é and è) + // This tests that character-to-byte offset conversion works with UTF-8 + assert!(!result.matches.is_empty()); + assert!(result.matches[0].line_content.contains("régulière")); +} + +#[test] +fn fuzzy_empty_query_returns_empty() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("a.txt", "some content\n")]); + + let parsed = parse_grep_query(""); + let result = picker.grep(&parsed, &fuzzy_opts()); + + // Empty query returns git-modified files, not fuzzy matches + assert_eq!(result.matches.len(), 0); +} + +#[test] +fn fuzzy_with_extension_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("a.rs", "use std::io;\nfn main() {}\n"), + ("b.txt", "use std::io;\nsome text\n"), + ("c.rs", "use std::fs;\n"), + ], + ); + + let parsed = parse_grep_query("use std *.rs"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + // Should only search .rs files + for file in &result.files { + assert!( + file.relative_path(&picker).ends_with(".rs"), + "should only match .rs files, got: {}", + file.relative_path(&picker) + ); + } +} + +#[test] +fn fuzzy_respects_page_limit() { + let tmp = TempDir::new().unwrap(); + let mut content = String::new(); + for i in 0..100 { + content.push_str(&format!("line {} target\n", i)); + } + fs::write(tmp.path().join("big.txt"), &content).unwrap(); + let picker = create_picker(tmp.path(), &[("big.txt", &content)]); + + let mut opts = fuzzy_opts(); + opts.page_limit = 10; + opts.max_matches_per_file = 50; + + let parsed = parse_grep_query("target"); + let result = picker.grep(&parsed, &opts); + + // page_limit is a soft minimum: we always finish the current file, so we + // get at least page_limit matches (no data loss) and at most + // max_matches_per_file (200) from a single file. + assert!( + result.matches.len() >= opts.page_limit, + "should return at least page_limit matches: got {}", + result.matches.len() + ); + assert!( + result.matches.len() <= opts.max_matches_per_file, + "should never exceed max_matches_per_file: got {}", + result.matches.len() + ); + + assert_eq!( + result.matches.len(), + opts.max_matches_per_file, + "all limit of lines must be returned" + ); +} + +#[test] +fn fuzzy_respects_max_matches_per_file() { + let tmp = TempDir::new().unwrap(); + let mut content = String::new(); + for i in 0..50 { + content.push_str(&format!("line {} match_target\n", i)); + } + fs::write(tmp.path().join("many.txt"), &content).unwrap(); + let picker = create_picker(tmp.path(), &[("many.txt", &content)]); + + let mut opts = fuzzy_opts(); + opts.max_matches_per_file = 5; + + let parsed = parse_grep_query("match"); + let result = picker.grep(&parsed, &opts); + + assert_eq!( + result.matches.len(), + 5, + "should cap at max_matches_per_file" + ); +} + +#[test] +fn fuzzy_filters_low_quality_matches() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "code.rs", + "fn mutex_lock() {}\nfn xyz() {}\nfn abc_def_ghi() {}\nfn abcdefghij() {}\n", + )], + ); + + // Search for "abc" - should match "abc_def_ghi" and "abcdefghij" with high scores, + // but NOT "xyz" (no relation) or "mutex_lock" (only weak letter overlap) + let parsed = parse_grep_query("abc"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + // Should only get high-quality matches + assert!( + result.matches.len() <= 2, + "should filter out low-quality fuzzy matches, got {} matches", + result.matches.len() + ); + + // All matches should contain reasonable character overlap + for m in &result.matches { + assert!( + m.line_content.contains("abc") || m.line_content.contains("abc_"), + "match '{}' should be high-quality", + m.line_content + ); + } +} + +#[test] +fn fuzzy_exact_match_always_passes() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[("test.txt", "exact match line\nno match here\n")], + ); + + // Exact matches should always pass regardless of score threshold + let parsed = parse_grep_query("exact"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + assert_eq!( + result.matches.len(), + 1, + "exact match should always pass score threshold" + ); + assert!(result.matches[0].line_content.contains("exact")); +} + +#[test] +fn fuzzy_score_is_captured() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("test.txt", "hello world\ngoodbye world\n")]); + + let parsed = parse_grep_query("hello"); + let result = picker.grep(&parsed, &fuzzy_opts()); + + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + + // Fuzzy score should be set (Some) for fuzzy mode matches + assert!( + m.fuzzy_score.is_some(), + "fuzzy_score should be set in fuzzy grep mode" + ); + assert!( + m.fuzzy_score.unwrap() > 0, + "fuzzy_score should be positive for a good match" + ); +} + +#[test] +fn fuzzy_score_is_none_in_plain_mode() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker(tmp.path(), &[("test.txt", "hello world\n")]); + + let parsed = parse_grep_query("hello"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + + // fuzzy_score should be None for plain text mode + assert!( + m.fuzzy_score.is_none(), + "fuzzy_score should be None in plain text mode" + ); +} + +/// Regression: memmem prefilter rejected files where content casing differed +/// from the query, even under smart_case. E.g. "vfio-kvm" failed to find +/// "VFIO-KVM" because the lowercased finder did a case-sensitive scan. +#[test] +fn plain_text_smart_case_finds_uppercase_content_with_lowercase_query() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[( + "driver.c", + "// VFIO-KVM integration\nstatic int init(void) {}\n", + )], + ); + + let parsed = parse_grep_query("vfio-kvm"); + let result = picker.grep(&parsed, &plain_opts()); + + assert_eq!( + result.matches.len(), + 1, + "lowercase query should case-insensitively match 'VFIO-KVM'" + ); +} diff --git a/crates/fff-core/tests/grep_overflow_path_constraint_segfault.rs b/crates/fff-core/tests/grep_overflow_path_constraint_segfault.rs new file mode 100644 index 0000000..0cad79c --- /dev/null +++ b/crates/fff-core/tests/grep_overflow_path_constraint_segfault.rs @@ -0,0 +1,57 @@ +use std::fs; + +use fff_search::file_picker::FilePicker; +use fff_search::grep::{GrepMode, GrepSearchOptions}; +use fff_search::{AiGrepConfig, FFFQuery, FilePickerOptions}; +use tempfile::TempDir; + +// bug pinning https://github.com/dmtrKovalenko/fff/issues/618 +#[test] +fn grep_path_constraint_on_overflow_file_does_not_segfault() { + let tmp = TempDir::new().expect("tempdir"); + let base = tmp.path(); + let spec_dir = base.join("specs"); + fs::create_dir_all(&spec_dir).expect("mkdir specs"); + + let file = spec_dir.join("annotation-plan.md"); + fs::write(&file, "dependency\n").expect("write test file"); + + // Intentionally do NOT call `collect_files()`. This leaves the base path + // arena unset/null. `handle_create_or_modify` then adds the file as an + // overflow file whose path chunks live in the overflow arena. + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + watch: false, + enable_home_dir_scanning: true, + ..Default::default() + }) + .expect("create picker"); + + let is_overflow = picker + .handle_create_or_modify(&file) + .expect("add overflow file") + .is_overflow(); + assert!(is_overflow, "file must be added to the overflow arena"); + + // Mirrors the `ffgrep({ pattern: "dependency", path: "specs/annotation-plan.md" })` + // call that crashed: in AI grep mode the path token becomes a FilePath + // constraint and `dependency` becomes the grep text. + let parsed = FFFQuery::parse("specs/annotation-plan.md dependency", AiGrepConfig); + + let opts = GrepSearchOptions { + mode: GrepMode::PlainText, + page_limit: 20, + smart_case: true, + ..Default::default() + }; + + // original issue: + // Debug builds typically abort with Rust's unsafe precondition check at + // simd_path.rs: ChunkedString::write_to_string. Release builds may SIGSEGV + // in memmove from an address like 0x30 (null arena + chunk_index * 16). + let result = picker.grep(&parsed, &opts); + + assert_eq!(result.files.len(), 1, "the overflow file should match"); + assert_eq!(result.matches.len(), 1, "`dependency` should match once"); +} diff --git a/crates/fff-core/tests/lmdb_stale_lock_deadlock.rs b/crates/fff-core/tests/lmdb_stale_lock_deadlock.rs new file mode 100644 index 0000000..a27e784 --- /dev/null +++ b/crates/fff-core/tests/lmdb_stale_lock_deadlock.rs @@ -0,0 +1,455 @@ +//! Reproduces the deadlock/hang caused by LMDB writer mutex contention. +//! +//! When another process holds the LMDB writer mutex (via a long-running write +//! transaction or because it crashed without releasing it), any call to +//! `write_txn()` blocks indefinitely — including on the neovim main thread +//! during `QueryTracker::open()` or frecency `track_access()`. +//! +//! In production this manifests as neovim hanging on startup: +//! require('fff.core').ensure_initialized() +//! → init_db() → QueryTracker::open() → write_txn() → HANGS +//! +//! Or during normal use when BufEnter fires: +//! track_access → frecency.track_access() → write_txn() → HANGS +//! +//! Reproduction: fork a child process that holds the LMDB write lock +//! indefinitely, then attempt to use the same database from the parent. +//! The parent's `write_txn()` blocks on the cross-process writer mutex. +//! +//! This test confirms that the current code has NO timeout or fallback when the +//! LMDB writer mutex is unavailable — making it vulnerable to indefinite hangs +//! whenever another process (fff-mcp, another neovim, or a crashed instance) +//! holds or has stuck the mutex. + +#![cfg(unix)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::time::Duration; + +use fff_search::frecency::FrecencyTracker; +use fff_search::query_tracker::QueryTracker; + +/// Returns whether `f` completes within `timeout`. +fn completes_within( + label: &'static str, + timeout: Duration, + f: impl FnOnce() + Send + 'static, +) -> bool { + let (tx, rx) = mpsc::channel::<()>(); + let _worker = std::thread::Builder::new() + .name(format!("deadlock-repro-{label}")) + .spawn(move || { + f(); + let _ = tx.send(()); + }) + .expect("spawn worker"); + + rx.recv_timeout(timeout).is_ok() +} + +/// Fork a child that opens the LMDB env and holds a write transaction +/// indefinitely (simulating a stuck/long-running process). Returns the +/// child PID so the parent can kill it during cleanup. +fn fork_child_holding_write_lock(db_path: &Path) -> libc::pid_t { + let db_path_str = db_path.to_str().unwrap().to_owned(); + + let mut pipe_fds: [libc::c_int; 2] = [0; 2]; + assert_eq!(unsafe { libc::pipe(pipe_fds.as_mut_ptr()) }, 0); + let read_fd = pipe_fds[0]; + let write_fd = pipe_fds[1]; + + let child_pid = unsafe { libc::fork() }; + match child_pid { + -1 => panic!("fork() failed: {}", std::io::Error::last_os_error()), + 0 => { + // === CHILD PROCESS === + unsafe { libc::close(read_fd) }; + + let env = unsafe { + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(10 * 1024 * 1024); + opts.open(Path::new(&db_path_str)).expect("child: open env") + }; + + // Acquire the cross-process writer mutex via write_txn + let _wtxn = env.write_txn().expect("child: write_txn"); + + // Signal parent that the lock is held + unsafe { libc::write(write_fd, b"R".as_ptr() as *const libc::c_void, 1) }; + + // Hold the lock forever — parent will eventually kill us + loop { + unsafe { libc::pause() }; + } + } + pid => { + // === PARENT PROCESS === + unsafe { libc::close(write_fd) }; + + // Wait for child to confirm it holds the write lock + let mut buf = [0u8; 1]; + let n = unsafe { libc::read(read_fd, buf.as_mut_ptr() as *mut libc::c_void, 1) }; + assert_eq!(n, 1, "child didn't signal readiness"); + assert_eq!(buf[0], b'R'); + unsafe { libc::close(read_fd) }; + + pid + } + } +} + +/// Kill and reap the child process. +fn kill_child(pid: libc::pid_t) { + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status: libc::c_int = 0; + libc::waitpid(pid, &mut status, 0); + } +} + +/// Verify QueryTracker works correctly after close+reopen — the +/// open_database_safe path must find existing named databases via read txn. +#[test] +fn lmdb_reopen_finds_existing_databases() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = tmp.path().join("lmdb_reopen"); + fs::create_dir_all(&db_path).unwrap(); + + // First open: creates the databases via write_txn fallback + { + let mut tracker = QueryTracker::open(&db_path).unwrap(); + let project = Path::new("/test/project"); + let file = Path::new("/test/project/src/main.rs"); + tracker + .track_query_completion("hello", project, file) + .unwrap(); + } + + // Second open: must find existing databases via read txn (no write_txn needed) + { + let tracker = QueryTracker::open(&db_path).unwrap(); + let project = Path::new("/test/project"); + let result = tracker.get_historical_query(project, 0).unwrap(); + assert_eq!( + result, + Some("hello".to_string()), + "Query history should persist across close/reopen" + ); + } +} + +/// Env var the test binary checks on startup. When set, the binary skips the +/// test harness and runs as a child worker instead. This avoids fork() in a +/// multi-threaded parent — which copies mutex/allocator state from threads +/// that no longer exist in the child and can deadlock heed/libc. +const CHILD_MODE_ENV: &str = "FFF_PARALLEL_OPEN_CLOSE_CHILD"; + +/// Runs before the test harness when `CHILD_MODE_ENV` is set. Re-exec of +/// the test binary lets us start child workers without forking from a +/// multi-threaded parent. +#[ctor::ctor] +fn maybe_enter_child_mode() { + if let Ok(spec) = std::env::var(CHILD_MODE_ENV) { + let code = run_child_from_spec(&spec); + std::process::exit(code); + } +} + +/// Spec format: `db_path|idx|iterations|writer(0|1)` +fn run_child_from_spec(spec: &str) -> i32 { + let parts: Vec<&str> = spec.split('|').collect(); + if parts.len() != 4 { + return CHILD_BAD_SPEC; + } + let db_path = parts[0]; + let idx: usize = match parts[1].parse() { + Ok(v) => v, + Err(_) => return CHILD_BAD_SPEC, + }; + let iterations: usize = match parts[2].parse() { + Ok(v) => v, + Err(_) => return CHILD_BAD_SPEC, + }; + let is_writer = parts[3] == "1"; + child_open_close_loop(db_path, iterations, idx, is_writer) +} + +const CHILD_OK: i32 = 0; +const CHILD_OPEN_FAILED: i32 = 10; +const CHILD_READ_FAILED: i32 = 11; +const CHILD_WRITE_FAILED: i32 = 12; +const CHILD_BAD_SPEC: i32 = 13; + +fn child_open_close_loop(db_path: &str, iterations: usize, idx: usize, is_writer: bool) -> i32 { + let project = Path::new("/test/project"); + for i in 0..iterations { + let tracker = match QueryTracker::open(Path::new(db_path)) { + Ok(t) => t, + Err(e) => { + eprintln!("child {idx} iter {i} reader open failed: {e:?}"); + return CHILD_OPEN_FAILED; + } + }; + if let Err(e) = tracker.get_historical_query(project, 0) { + eprintln!("child {idx} iter {i} read failed: {e:?}"); + return CHILD_READ_FAILED; + } + drop(tracker); + + if is_writer { + let mut tracker = match QueryTracker::open(Path::new(db_path)) { + Ok(t) => t, + Err(e) => { + eprintln!("child {idx} iter {i} writer open failed: {e:?}"); + return CHILD_OPEN_FAILED; + } + }; + let file = PathBuf::from(format!("/test/project/c{idx}_{i}.rs")); + if let Err(e) = tracker.track_query_completion(&format!("q{idx}_{i}"), project, &file) { + eprintln!("child {idx} iter {i} write failed: {e:?}"); + return CHILD_WRITE_FAILED; + } + drop(tracker); + } + } + CHILD_OK +} + +/// Spawn `n` child processes via `Command::new(current_exe)`. No fork, so +/// mutex/allocator state is not inherited. `writers` children also issue +/// writes; the rest only read. +fn spawn_open_close_children( + db_path: &Path, + n: usize, + writers: usize, + ops_per_child: usize, +) -> Vec { + assert!(writers <= n); + let exe = std::env::current_exe().expect("current_exe"); + let db_path_str = db_path.to_str().unwrap().to_owned(); + + (0..n) + .map(|idx| { + let is_writer = idx < writers; + let spec = format!( + "{db_path_str}|{idx}|{ops_per_child}|{}", + if is_writer { 1 } else { 0 } + ); + std::process::Command::new(&exe) + .env(CHILD_MODE_ENV, spec) + .env_remove("RUST_LOG") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .expect("spawn child") + }) + .collect() +} + +/// Wait for every child with a per-call deadline. On timeout, kill and reap +/// remaining children and return an Err describing the stuck set. +fn wait_all_with_deadline( + mut children: Vec, + deadline: std::time::Instant, +) -> Result<(), String> { + let mut failures: Vec<(u32, Option)> = Vec::new(); + let mut remaining: Vec = Vec::new(); + + for mut child in children.drain(..) { + loop { + match child.try_wait() { + Ok(Some(status)) => { + let code = status.code(); + if code != Some(CHILD_OK) { + failures.push((child.id(), code)); + } + break; + } + Ok(None) => { + if std::time::Instant::now() >= deadline { + remaining.push(child); + break; + } + std::thread::sleep(Duration::from_millis(25)); + } + Err(e) => { + failures.push((child.id(), None)); + let _ = e; + break; + } + } + } + } + + if !remaining.is_empty() { + let stuck: Vec = remaining.iter().map(|c| c.id()).collect(); + for child in &mut remaining { + let _ = child.kill(); + let _ = child.wait(); + } + return Err(format!( + "deadline exceeded; children still running: {stuck:?}" + )); + } + + if !failures.is_empty() { + return Err(format!("children failed: {failures:?}")); + } + Ok(()) +} + +/// Many processes open/close `QueryTracker` against the same DB path. +/// Readers only: seeds once, then spawns N reader children. +/// +/// heed 0.22 forbids opening the same env twice *within* one process +/// (EnvAlreadyOpened), so cross-process contention is the right axis. +#[test] +fn query_tracker_many_parallel_open_close_same_path_readers() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = tmp.path().join("parallel_open_close_readers"); + fs::create_dir_all(&db_path).unwrap(); + + { + let mut tracker = QueryTracker::open(&db_path).unwrap(); + tracker + .track_query_completion( + "seed", + Path::new("/test/project"), + Path::new("/test/project/src/main.rs"), + ) + .unwrap(); + } + + const N: usize = 8; + const OPS: usize = 4; + + let children = spawn_open_close_children(&db_path, N, 0, OPS); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + wait_all_with_deadline(children, deadline).expect("parallel open/close (readers)"); + + let tracker = QueryTracker::open(&db_path).expect("post-storm reopen"); + let project = Path::new("/test/project"); + let result = tracker.get_historical_query(project, 0).unwrap(); + assert_eq!( + result, + Some("seed".to_string()), + "Seed query should still be readable after parallel open/close storm" + ); +} + +/// Stronger variant: multiple processes race opens that both read AND write. +/// LMDB serializes writers via a cross-process mutex; test that serialization +/// makes forward progress and open/close pairs don't deadlock. +#[test] +fn query_tracker_parallel_open_write_close_same_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = tmp.path().join("parallel_open_write_close"); + fs::create_dir_all(&db_path).unwrap(); + + { + let mut tracker = QueryTracker::open(&db_path).unwrap(); + tracker + .track_query_completion( + "seed", + Path::new("/test/project"), + Path::new("/test/project/src/main.rs"), + ) + .unwrap(); + } + + const N: usize = 4; + const WRITERS: usize = 4; + const OPS: usize = 3; + + let children = spawn_open_close_children(&db_path, N, WRITERS, OPS); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + wait_all_with_deadline(children, deadline).expect("parallel open/write/close"); + + let tracker = QueryTracker::open(&db_path).expect("post-storm reopen"); + let project = Path::new("/test/project"); + let seed = tracker.get_historical_query(project, 0).unwrap(); + assert!( + seed.is_some(), + "Env unreadable after parallel open/write/close storm" + ); +} + +/// Within a single process, opening the same env path twice concurrently is +/// forbidden by heed — but a strict sequential open→use→drop→open loop must +/// succeed every iteration. Regression guard for the reopen path. +#[test] +fn query_tracker_sequential_reopen_loop_same_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = tmp.path().join("sequential_reopen_loop"); + fs::create_dir_all(&db_path).unwrap(); + + { + let mut tracker = QueryTracker::open(&db_path).unwrap(); + tracker + .track_query_completion( + "seed", + Path::new("/test/project"), + Path::new("/test/project/src/main.rs"), + ) + .unwrap(); + } + + for i in 0..64 { + let mut tracker = QueryTracker::open(&db_path).expect("sequential reopen"); + let project = Path::new("/test/project"); + let file = PathBuf::from(format!("/test/project/iter_{i}.rs")); + tracker + .track_query_completion(&format!("iter_{i}"), project, &file) + .expect("sequential track"); + drop(tracker); + } + + let tracker = QueryTracker::open(&db_path).expect("final reopen"); + let project = Path::new("/test/project"); + assert!(tracker.get_historical_query(project, 0).unwrap().is_some()); +} + +/// When the frecency DB doesn't exist yet, `FrecencyTracker::open()` falls +/// through to `write_txn()` + `create_database()`. This blocks if another +/// process holds the writer mutex. This is the first-launch path. +/// +/// NOTE: this test is disabled because heed 0.22 appears to use a +/// try-then-create pattern for unnamed databases that doesn't always block. +/// The QueryTracker test above (named databases, always needs write_txn) +/// reliably demonstrates the same underlying issue. +#[test] +#[ignore = "heed 0.22 unnamed db creation may not require writer mutex in all cases"] +fn frecency_open_blocks_on_fresh_db_when_another_process_holds_write_lock() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = tmp.path().join("frecency_fresh_deadlock"); + fs::create_dir_all(&db_path).unwrap(); + + let env = unsafe { + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(10 * 1024 * 1024); + opts.open(&db_path).unwrap() + }; + drop(env); + + let child_pid = fork_child_holding_write_lock(&db_path); + + let db_path_clone = db_path.clone(); + let completed = completes_within( + "FrecencyTracker::open (fresh db) while writer held", + Duration::from_secs(3), + move || { + let _result = FrecencyTracker::open(&db_path_clone); + }, + ); + + kill_child(child_pid); + + assert!( + !completed, + "Expected FrecencyTracker::open() on a fresh DB to block (writer mutex \ + held by another process), but it completed." + ); +} diff --git a/crates/fff-core/tests/new_directory_watcher_test.rs b/crates/fff-core/tests/new_directory_watcher_test.rs new file mode 100644 index 0000000..9cb9fc1 --- /dev/null +++ b/crates/fff-core/tests/new_directory_watcher_test.rs @@ -0,0 +1,539 @@ +//! Integration test: verifying that the background watcher dynamically detects +//! newly created directories and picks up files written inside them. +//! +//! This covers the NonRecursive watching behavior where: +//! 1. The watcher starts with watches on directories discovered during the +//! initial scan. +//! 2. A brand-new subdirectory is created at runtime (after the scan). +//! 3. The watcher's event handler detects the directory Create event, +//! collects it, and sends it to the owner thread via `watch_tx`. +//! 4. The owner thread adds a NonRecursive watch on the new directory and +//! does a flat (non-recursive) read_dir to inject files that already +//! exist (race-window coverage). +//! 5. Files created *after* the watch is established are picked up via +//! normal event delivery. +//! +//! The test uses the real `BackgroundWatcher` (via `watch: true`) and polls +//! the picker until the expected files appear or a timeout expires. + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{ + FilePickerOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency, +}; + +// ═══════════════════════════════════════════════════════════════════════ +// Helpers +// ═══════════════════════════════════════════════════════════════════════ + +fn git_run(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .output() + .unwrap_or_else(|e| panic!("git {:?} failed: {}", args, e)); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git_init_and_commit(dir: &Path) { + git_run(dir, &["init", "-b", "main"]); + git_run(dir, &["add", "-A"]); + git_run(dir, &["commit", "-m", "initial"]); +} + +fn make_watched_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::noop(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + (shared_picker, shared_frecency) +} + +/// Wait for the initial scan + watcher to be fully ready. +fn wait_ready(shared_picker: &SharedFilePicker) { + assert!( + shared_picker.wait_for_scan(Duration::from_secs(10)), + "Timed out waiting for initial scan" + ); + assert!( + shared_picker.wait_for_watcher(Duration::from_secs(10)), + "Timed out waiting for watcher" + ); +} + +/// Poll the picker until `predicate` returns true or timeout expires. +/// Returns the elapsed duration if successful, panics on timeout. +fn poll_until( + shared_picker: &SharedFilePicker, + timeout: Duration, + description: &str, + predicate: impl Fn(&FilePicker) -> bool, +) -> Duration { + let start = Instant::now(); + loop { + { + let guard = shared_picker.read().unwrap(); + if let Some(ref picker) = *guard { + if predicate(picker) { + return start.elapsed(); + } + } + } + if start.elapsed() >= timeout { + // One final attempt to give a useful error message. + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let file_count = picker.get_files().len(); + let paths: Vec = picker + .get_files() + .iter() + .map(|f| f.relative_path(picker)) + .collect(); + panic!( + "Timed out after {:?} waiting for: {}\n\ + Current file count: {}\n\ + Current files: {:?}", + timeout, description, file_count, paths + ); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn grep_plain_count(picker: &FilePicker, query: &str) -> usize { + let parsed = parse_grep_query(query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 500, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + picker.grep(&parsed, &opts).matches.len() +} + +fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + fff_search::FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +/// Debounce timeout in the watcher is 250ms. Events need to propagate through +/// the debouncer, the owner thread park loop (1s), and the picker write lock. +/// We use a generous timeout for CI environments. +const WATCHER_TIMEOUT: Duration = Duration::from_secs(10); + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +/// Create a new directory and immediately write a file inside it. +/// The file is written before the watch is registered, so the flat +/// inject_existing_files scan in the owner thread must catch it. +#[test] +fn new_directory_and_file_detected_by_watcher() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + // Seed the repo with some initial files so the scan has something. + fs::create_dir_all(base.join("src")).unwrap(); + fs::write( + base.join("src/main.rs"), + "fn main() { println!(\"INITIAL_MARKER\"); }\n", + ) + .unwrap(); + fs::write(base.join("README.md"), "# Test project\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Sanity: initial file is indexed. + poll_until( + &shared_picker, + Duration::from_secs(5), + "initial file src/main.rs indexed", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("main.rs")) + }, + ); + + // Create a new directory and write a file into it immediately. + // The file exists before the watch is registered — inject_existing_files + // in the owner thread catches it via a flat read_dir. + let new_dir = base.join("src/components"); + fs::create_dir_all(&new_dir).unwrap(); + fs::write( + new_dir.join("button.rs"), + "pub struct Button;\nconst TOKEN: &str = \"NEW_DIR_BUTTON_TOKEN\";\n", + ) + .unwrap(); + + // Wait for the watcher to detect the new directory + file. + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "file src/components/button.rs in new directory", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("button.rs")) + }, + ); + eprintln!( + " New directory + file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Also verify via grep that the content is accessible. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds NEW_DIR_BUTTON_TOKEN", + |picker| grep_plain_count(picker, "NEW_DIR_BUTTON_TOKEN") >= 1, + ); + + // And via fuzzy search. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "fuzzy search finds button.rs", + |picker| { + let results = fuzzy_search_paths(picker, "button"); + results.iter().any(|p| p.contains("button.rs")) + }, + ); +} + +/// Create a new directory, then create files AFTER a delay to ensure the +/// watch was established on the directory. +#[test] +fn file_created_after_directory_watch_established() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::create_dir_all(base.join("lib")).unwrap(); + fs::write(base.join("lib/utils.rs"), "pub fn helper() {}\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create the directory first, wait for the watcher to register it. + let new_dir = base.join("lib/models"); + fs::create_dir(&new_dir).unwrap(); + + // Wait long enough for the debouncer to flush + owner thread to add watch. + std::thread::sleep(Duration::from_millis(2000)); + + // Now write a file into the already-watched directory. + fs::write( + new_dir.join("user.rs"), + "pub struct User { name: String }\nconst TOKEN: &str = \"POST_WATCH_USER_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "file lib/models/user.rs created after directory watch", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("user.rs")) + }, + ); + eprintln!( + " Post-watch file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Grep sanity. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds POST_WATCH_USER_TOKEN", + |picker| grep_plain_count(picker, "POST_WATCH_USER_TOKEN") >= 1, + ); +} + +/// Create a deeply nested directory tree all at once with create_dir_all +/// and write a file at the leaf. The watcher must detect the top-level +/// directory via the parent's watch, inject_existing_files finds the file +/// at the leaf (and intermediate dirs get their own watches from Create +/// events on subsequent levels). +#[test] +fn deeply_nested_new_directories_detected() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::write(base.join("root.txt"), "root file\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create each level one at a time, waiting for each watch to register. + // inject_existing_files is flat (non-recursive), so deeply nested dirs + // need each parent to be watched before we can see files at the leaf. + fs::create_dir(base.join("app")).unwrap(); + std::thread::sleep(Duration::from_millis(2000)); + + fs::create_dir(base.join("app/services")).unwrap(); + std::thread::sleep(Duration::from_millis(2000)); + + fs::create_dir(base.join("app/services/auth")).unwrap(); + // Write the file immediately — inject_existing_files catches it. + fs::write( + base.join("app/services/auth/jwt.rs"), + "pub fn verify_token() {}\nconst TOKEN: &str = \"DEEP_NESTED_JWT_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "deeply nested file app/services/auth/jwt.rs", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("jwt.rs")) + }, + ); + eprintln!( + " Deeply nested file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + // Verify content is grepable. + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds DEEP_NESTED_JWT_TOKEN", + |picker| grep_plain_count(picker, "DEEP_NESTED_JWT_TOKEN") >= 1, + ); + + // Now create a sibling at the same depth — the parent (app/services) + // is already watched, so this just needs the flat inject. + let sibling_dir = base.join("app/services/database"); + fs::create_dir(&sibling_dir).unwrap(); + fs::write( + sibling_dir.join("pool.rs"), + "pub struct ConnectionPool;\nconst TOKEN: &str = \"SIBLING_POOL_TOKEN\";\n", + ) + .unwrap(); + + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "sibling nested file app/services/database/pool.rs", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("pool.rs")) + }, + ); + eprintln!( + " Sibling nested file detected in {:.0}ms", + elapsed.as_secs_f64() * 1000.0 + ); + + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "grep finds SIBLING_POOL_TOKEN", + |picker| grep_plain_count(picker, "SIBLING_POOL_TOKEN") >= 1, + ); +} + +/// Create a new directory and immediately burst-write multiple files. +/// inject_existing_files catches all of them in one flat read_dir. +#[test] +fn burst_file_creation_in_new_directory() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::create_dir_all(base.join("src")).unwrap(); + fs::write(base.join("src/lib.rs"), "// lib\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create a new directory and immediately write 5 files. + let batch_dir = base.join("src/batch"); + fs::create_dir(&batch_dir).unwrap(); + + let file_count = 5; + for i in 0..file_count { + fs::write( + batch_dir.join(format!("item_{i}.rs")), + format!("pub struct Item{i};\nconst TOKEN: &str = \"BATCH_ITEM_{i}\";\n"), + ) + .unwrap(); + } + + // Wait for ALL files to appear. + let elapsed = poll_until( + &shared_picker, + WATCHER_TIMEOUT, + &format!("all {file_count} batch files in src/batch/"), + |picker| { + let batch_count = picker + .get_files() + .iter() + .filter(|f| { + let p = f.relative_path(picker); + p.starts_with("src/batch/") || p.starts_with("src\\batch\\") + }) + .count(); + batch_count >= file_count + }, + ); + eprintln!( + " All {} burst files detected in {:.0}ms", + file_count, + elapsed.as_secs_f64() * 1000.0 + ); + + // Verify each file's content is grepable. + for i in 0..file_count { + let token = format!("BATCH_ITEM_{i}"); + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + &format!("grep finds {token}"), + |picker| grep_plain_count(picker, &token) >= 1, + ); + } +} + +/// Verify that gitignored directories created at runtime are NOT watched +/// and their files do NOT appear in the index. +#[test] +fn gitignored_new_directory_excluded() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().canonicalize().unwrap(); + + fs::write(base.join("main.rs"), "fn main() {}\n").unwrap(); + // Ignore the build/ directory. + fs::write(base.join(".gitignore"), "build/\n").unwrap(); + + git_init_and_commit(&base); + + let (shared_picker, _frecency) = make_watched_picker(&base); + wait_ready(&shared_picker); + + // Create a gitignored directory with files. + let ignored_dir = base.join("build"); + fs::create_dir(&ignored_dir).unwrap(); + fs::write( + ignored_dir.join("output.rs"), + "const TOKEN: &str = \"IGNORED_BUILD_TOKEN\";\n", + ) + .unwrap(); + + // Also create a non-ignored directory to confirm the watcher works. + let good_dir = base.join("src"); + fs::create_dir(&good_dir).unwrap(); + fs::write( + good_dir.join("app.rs"), + "const TOKEN: &str = \"GOOD_SRC_TOKEN\";\n", + ) + .unwrap(); + + // Wait for the non-ignored file to appear (proves watcher is working). + poll_until( + &shared_picker, + WATCHER_TIMEOUT, + "non-ignored file src/app.rs appears", + |picker| { + picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("app.rs")) + }, + ); + + // Give extra time for any straggler events from the ignored dir. + std::thread::sleep(Duration::from_secs(2)); + + // The gitignored file must NOT be in the index. + { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let has_ignored = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).contains("output.rs")); + assert!( + !has_ignored, + "Gitignored file build/output.rs should NOT be in the index" + ); + + let grep_count = grep_plain_count(picker, "IGNORED_BUILD_TOKEN"); + assert_eq!(grep_count, 0, "Gitignored content should NOT be grepable"); + } +} diff --git a/crates/fff-core/tests/overflow_frecency_segfault.rs b/crates/fff-core/tests/overflow_frecency_segfault.rs new file mode 100644 index 0000000..1a9af6a --- /dev/null +++ b/crates/fff-core/tests/overflow_frecency_segfault.rs @@ -0,0 +1,93 @@ +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::{ + FFFQuery, FilePickerOptions, FileSearchConfig, FrecencyTracker, FuzzySearchOptions, + SharedFilePicker, SharedFrecency, +}; +use std::fs; +use std::path::Path; +use std::time::Duration; +use tempfile::TempDir; + +// regression pinning test: if base count=0 updating frecency should not panic +#[test] +fn update_single_file_frecency_on_overflow_file_does_not_segfault() { + let base = TempDir::new().expect("mktemp base"); + let db = TempDir::new().expect("mktemp db"); + + // EMPTY base tree. The initial scan indexes zero files, so `base_count == 0` + // and the base path arena is an empty store whose pointer is the dangling, + // 16-byte-aligned `0x10`. This is the exact state observed at the crash: + // base_count=0 is_overflow=true base_arena=0x10 overflow_arena=0x9f5400000 + let base_path = base.path().canonicalize().expect("canonicalize base"); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + let tracker = FrecencyTracker::open(db.path().join("frecency.mdb")).expect("open frecency db"); + shared_frecency.init(tracker).expect("init frecency"); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base_path.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: false, + // AI mode is the mode that drives per-file frecency updates from the + // watcher; we exercise the same picker method directly. + mode: FFFMode::Ai, + // No watcher: we add an overflow file and update frecency by hand so + // the repro is deterministic and timing-independent. + watch: false, + ..Default::default() + }, + ) + .expect("FilePicker::new_with_shared_state"); + + assert!( + shared_picker.wait_for_scan(Duration::from_secs(10)), + "initial scan did not complete" + ); + + // Create ONE new file after the (empty) scan. It is added to the overflow + // arena (`FileItem::is_overflow()` is true), with valid overflow chunk + // indices into the overflow arena. + let new_file = base_path.join("created_after_scan.txt"); + fs::write(&new_file, "hello\n").unwrap(); + + { + let mut guard = shared_picker.write().expect("picker write lock"); + let picker: &mut FilePicker = guard.as_mut().expect("picker initialized"); + picker.handle_create_or_modify(&new_file); + } + + // simulate watcher events + { + let frecency_guard = shared_frecency.read().expect("frecency read lock"); + let frecency = frecency_guard.as_ref().expect("frecency initialized"); + + let mut guard = shared_picker.write().expect("picker write lock"); + let picker: &mut FilePicker = guard.as_mut().expect("picker initialized"); + + let _ = frecency.track_access(new_file.as_path()); + let _ = picker.update_single_file_frecency(new_file.as_path(), frecency); + } + + // Reaching here means the overflow file was read through the correct arena. + let _: &Path = base_path.as_path(); + + { + let mut guard = shared_picker.write().expect("picker write lock"); + let picker: &mut FilePicker = guard.as_mut().expect("picker initialized"); + let results = picker.fuzzy_search( + &FFFQuery::parse("created_after_scan *.txt", FileSearchConfig), + None, + FuzzySearchOptions::default(), + ); + + assert_eq!(results.total_matched, 1); + } + + drop(shared_picker); + drop(shared_frecency); +} diff --git a/crates/fff-core/tests/path_separator_constraint_test.rs b/crates/fff-core/tests/path_separator_constraint_test.rs new file mode 100644 index 0000000..9519ef8 --- /dev/null +++ b/crates/fff-core/tests/path_separator_constraint_test.rs @@ -0,0 +1,248 @@ +//! Regression test for https://github.com/dmtrKovalenko/fff/issues/381 +//! +//! Directory (`PathSegment`) and file-path (`FilePath`) constraints must +//! return results on every platform. Indexed paths on Windows use native +//! backslash separators, so constraint matching has to accept either `/` +//! or `\\` as a path boundary. + +use std::fs; +use std::path::Path; +use tempfile::TempDir; + +use fff_search::file_picker::FilePicker; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{Constraint, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser}; + +fn create_picker(base: &Path, specs: &[(&str, &str)]) -> FilePicker { + for (rel, contents) in specs { + let full_path = base.join(rel); + if let Some(parent) = full_path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(&full_path, contents).unwrap(); + } + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + watch: false, + ..Default::default() + }) + .expect("failed to create FilePicker"); + picker.collect_files().expect("failed to collect files"); + picker +} + +fn plain_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1, + pagination: PaginationArgs { + offset: 0, + limit: 200, + }, + ..Default::default() + }, + ); + result + .items + .iter() + .map(|f| f.relative_path(picker)) + .collect() +} + +/// Treat a relative path as a sequence of components regardless of the +/// native separator so assertions are portable across Linux, macOS, Windows. +fn has_segment(path: &str, segment: &str) -> bool { + path.split(['/', '\\']).any(|s| s == segment) +} + +/// `grep handleRequest src/` — PathSegment constraint must match a nested +/// `src` directory on every platform. +#[test] +fn grep_with_path_segment_constraint_nested() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("app/modules/src/services/handler.lua", "handleRequest()\n"), + ("app/modules/lib/util.lua", "handleRequest()\n"), + ("src/main.rs", "fn handleRequest() {}\n"), + ], + ); + + let parsed = parse_grep_query("handleRequest src/"); + let result = picker.grep(&parsed, &plain_opts()); + + let matched_paths: Vec = result + .files + .iter() + .map(|f| f.relative_path(&picker)) + .collect(); + assert_eq!( + result.matches.len(), + 2, + "expected matches in two src/ files, got {matched_paths:?}" + ); + for p in &matched_paths { + assert!( + has_segment(p, "src"), + "every matched file must live under a `src` segment, got {p:?}" + ); + } +} + +/// `multi_grep` with a `PathSegment` constraint. +#[test] +fn multi_grep_with_path_segment_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ( + "app/modules/src/controller.lua", + "handleRequest\nprocessJob\n", + ), + ("app/modules/lib/helper.lua", "handleRequest\n"), + ("app/src/legacy.lua", "processJob\n"), + ], + ); + + let constraints = [Constraint::PathSegment("src")]; + let patterns = ["handleRequest", "processJob"]; + let result = picker.multi_grep(&patterns, &constraints, &plain_opts()); + + assert!( + !result.matches.is_empty(), + "multi_grep with `src/` constraint should return matches" + ); + + let matched_paths: Vec = result + .files + .iter() + .map(|f| f.relative_path(&picker)) + .collect(); + for p in &matched_paths { + assert!( + has_segment(p, "src"), + "every matched file must live under a `src` segment, got {p:?}" + ); + } + assert!(matched_paths.iter().any(|p| p.contains("controller.lua"))); + assert!(matched_paths.iter().any(|p| p.contains("legacy.lua"))); +} + +/// Fuzzy search (`find_files src/ Controller`) must apply the path-segment +/// filter to paths stored during indexing. +#[test] +fn fuzzy_search_with_path_segment_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("app/modules/src/services/BaseController.lua", "base\n"), + ("app/modules/src/services/UserController.lua", "user\n"), + ("app/modules/lib/BaseController.lua", "lib base\n"), + ("tests/src/MockController.lua", "mock\n"), + ], + ); + + let results = fuzzy_search_paths(&picker, "src/ Controller"); + + assert!( + !results.is_empty(), + "fuzzy search with `src/` constraint should return results" + ); + for p in &results { + assert!( + has_segment(p, "src"), + "every result must live under `src`, got {p:?}" + ); + } + assert!(results.iter().any(|p| p.contains("BaseController"))); + assert!(results.iter().any(|p| p.contains("UserController"))); + assert!(results.iter().any(|p| p.contains("MockController"))); +} + +/// `FilePath` suffix constraint must match stored paths even when components +/// are separated by the platform-native separator during indexing. +#[test] +fn multi_grep_with_file_path_suffix_constraint() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("app/modules/src/services/handler.lua", "handleRequest\n"), + ("other/src/services/handler.lua", "handleRequest\n"), + ("app/modules/src/services/other.lua", "handleRequest\n"), + ], + ); + + let constraints = [Constraint::FilePath("services/handler.lua")]; + let patterns = ["handleRequest"]; + let result = picker.multi_grep(&patterns, &constraints, &plain_opts()); + + let paths: Vec = result + .files + .iter() + .map(|f| f.relative_path(&picker)) + .collect(); + assert_eq!( + paths.len(), + 2, + "expected two matches for services/handler.lua, got {paths:?}" + ); + for p in &paths { + let ends_with_services_handler = + p.ends_with("services/handler.lua") || p.ends_with("services\\handler.lua"); + assert!( + ends_with_services_handler, + "matched path must end with services/handler.lua, got {p:?}" + ); + } +} + +/// Glob constraints must match native Windows paths — the picker normalises +/// separators when handing paths to the glob matcher. +#[test] +fn fuzzy_search_with_glob_constraint_matches_on_windows_paths() { + let tmp = TempDir::new().unwrap(); + let picker = create_picker( + tmp.path(), + &[ + ("app/src/components/Button.lua", "\n"), + ("app/src/services/handler.lua", "\n"), + ("app/lib/components/Ignored.lua", "\n"), + ], + ); + + let results = fuzzy_search_paths(&picker, "**/src/**/*.lua"); + assert!( + results.iter().any(|p| p.contains("Button.lua")), + "glob `**/src/**/*.lua` must match files below any `src/`, got {results:?}" + ); + assert!( + results.iter().any(|p| p.contains("handler.lua")), + "glob `**/src/**/*.lua` must match services/handler.lua, got {results:?}" + ); +} diff --git a/crates/fff-core/tests/real_binary_fixtures.rs b/crates/fff-core/tests/real_binary_fixtures.rs new file mode 100644 index 0000000..e4da069 --- /dev/null +++ b/crates/fff-core/tests/real_binary_fixtures.rs @@ -0,0 +1,214 @@ +//! Real-world binary fixture regression. +//! +//! Reproduces the exact bug chain we hit with `codex_view` (4.5 MB ELF, no +//! extension) and `codex_view.codex` (127 KB, unknown extension): both are +//! binary by content but slip past extension-only triage, so a plain grep +//! used to surface their NUL-laden bytes as "text" matches. +//! +//! The fixtures live in `tests/fixtures/binaries/`. `MARKER` is a string that +//! is present (as raw bytes) in BOTH binaries — the test first asserts that, +//! then drops the two binaries plus a single plain-text file containing the +//! same marker into a closed temp dir and greps for it. Only the text file may +//! come back; if binary detection ever regresses, a binary file re-enters the +//! results and this test fails. + +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency}; + +const MARKER: &str = "__jai_runtime_init"; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/binaries") +} + +fn plain_opts() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 200, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + } +} + +#[test] +fn real_binary_fixtures_are_detected_and_excluded_from_grep() { + let fixtures = fixtures_dir(); + let large = fixtures.join("codex_view"); // 4.5 MB ELF, no extension (> 2 MB) + let small = fixtures.join("codex_view.codex"); // 127 KB, unknown extension (< 2 MB) + + assert!( + large.exists() && small.exists(), + "missing binary fixtures in {}", + fixtures.display() + ); + + // Both fixtures must really contain the marker bytes, otherwise the grep + // exclusion assertion below would be vacuous. + let large_bytes = fs::read(&large).unwrap(); + let small_bytes = fs::read(&small).unwrap(); + assert!( + contains_subslice(&large_bytes, MARKER.as_bytes()), + "fixture codex_view no longer contains the marker {MARKER:?}" + ); + assert!( + contains_subslice(&small_bytes, MARKER.as_bytes()), + "fixture codex_view.codex no longer contains the marker {MARKER:?}" + ); + // Sanity on the size split that drives the two distinct code paths. + assert!( + large_bytes.len() > 2 * 1024 * 1024, + "codex_view must exceed the 2 MB non-indexable threshold" + ); + assert!( + small_bytes.len() < 2 * 1024 * 1024, + "codex_view.codex must stay under the 2 MB bigram cap" + ); + + // Closed environment: the two real binaries + one plain-text file that + // legitimately contains the marker. + let tmp = tempfile::TempDir::new().unwrap(); + let base = tmp.path(); + fs::copy(&large, base.join("codex_view")).unwrap(); + fs::copy(&small, base.join("codex_view.codex")).unwrap(); + fs::write( + base.join("marker.txt"), + format!("the only legitimate hit lives here: {MARKER}\n"), + ) + .unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("failed to create FilePicker"); + + assert!( + shared_picker.wait_for_indexing_complete(Duration::from_secs(10)), + "indexing/post-scan did not complete in time — binary classification may not have run yet" + ); + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + // Both binaries must be classified binary. + for name in ["codex_view", "codex_view.codex"] { + let flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).ends_with(name) && f.is_binary()); + assert!(flagged, "{name} must be flagged is_binary"); + } + + // we need to make sure that marker.txt ONLY can match as we have to match + // grep as binaries are excluded from the matching process + let parsed = parse_grep_query(MARKER); + let result = picker.grep(&parsed, &plain_opts()); + + let matched: Vec = result + .files + .iter() + .map(|f| f.relative_path(picker)) + .collect(); + + assert_eq!( + result.files.len(), + 1, + "exactly one file should match {MARKER:?}, got: {matched:?}" + ); + assert!( + matched[0].ends_with("marker.txt"), + "the only match must be marker.txt, got {:?}", + matched[0] + ); +} + +/// Tiny substring search over raw bytes (the marker may be surrounded by NULs). +fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool { + if needle.is_empty() || haystack.len() < needle.len() { + return false; + } + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +/// Deterministic regression for the Windows-CI failure where `codex_view` +/// (a >2 MB no-extension binary) was not flagged `is_binary`. Root cause was a +/// readiness-signal gap: `scanning` was cleared before `post_scan_indexing_active` +/// was set, so `wait_for_indexing_complete` could return before the binary sniff +/// ran. Uses synthetic fixtures (no repo/fixture dependency) covering both the +/// >2 MB non-indexable sniff path and the <2 MB bigram path, repeated to stress +/// the signal ordering. With the fix it must pass every iteration. +#[test] +fn binary_classification_done_before_indexing_wait_returns() { + const ITERATIONS: usize = 8; + // NUL bytes => `detect_binary_content` classifies as binary on every path. + let large = vec![0u8; 3 * 1024 * 1024]; // > 2 MB -> non-indexable sniff + let small = vec![0u8; 64 * 1024]; // < 2 MB -> bigram path + + for iteration in 0..ITERATIONS { + let tmp = tempfile::TempDir::new().unwrap(); + let base = tmp.path(); + fs::write(base.join("large_binary_no_ext"), &large).unwrap(); + fs::write(base.join("small.unknownext"), &small).unwrap(); + fs::write(base.join("readme.txt"), "hello world\n").unwrap(); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: true, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("failed to create FilePicker"); + + assert!( + shared_picker.wait_for_indexing_complete(Duration::from_secs(10)), + "iteration {iteration}: indexing/post-scan did not complete in time" + ); + + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + for name in ["large_binary_no_ext", "small.unknownext"] { + let flagged = picker + .get_files() + .iter() + .any(|f| f.relative_path(picker).ends_with(name) && f.is_binary()); + assert!( + flagged, + "iteration {iteration}: {name} must be flagged is_binary once \ + wait_for_indexing_complete returns" + ); + } + } +} diff --git a/crates/fff-core/tests/watcher_stop_under_lock.rs b/crates/fff-core/tests/watcher_stop_under_lock.rs new file mode 100644 index 0000000..42bc9fe --- /dev/null +++ b/crates/fff-core/tests/watcher_stop_under_lock.rs @@ -0,0 +1,165 @@ +//! Regression test: stopping the background watcher while the caller +//! holds the [`SharedFilePicker`] write lock must NOT deadlock. +//! +//! There are two lock-ordering hazards the watcher has to handle: +//! +//! 1. The debouncer's event thread calls our handler, which wants +//! `shared_picker.write()` to apply events. `stop()` used to +//! `join()` that thread under the caller's write guard. +//! +//! 2. The owner thread registers new-directory watches and injects +//! their existing files. Previously it held the debouncer mutex +//! across `shared_picker.write()`, while `stop()` takes the +//! debouncer mutex under the caller's write guard — inverse +//! lock orders, classic deadlock. +//! +//! macOS FSEvents is the reliable reproducer for (1) because fresh +//! `fs::write()` calls inside a just-watched temp dir queue events +//! faster than the debounce tick can drain them. Creating new +//! subdirectories exercises (2) via the owner thread's `watch_tx`. + +use std::fs; +use std::sync::mpsc; +use std::time::Duration; +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency}; + +/// Run `f` on a worker thread, require it to finish within `timeout`, +/// panic with `msg` otherwise. The caller gets to describe what the +/// worker is doing so a hung test produces an actionable message. +fn run_with_deadlock_guard( + msg: &'static str, + timeout: Duration, + f: impl FnOnce() + Send + 'static, +) { + let (done_tx, done_rx) = mpsc::channel::<()>(); + let worker = std::thread::Builder::new() + .name("deadlock-guard-worker".into()) + .spawn(move || { + f(); + let _ = done_tx.send(()); + }) + .expect("spawn worker"); + + match done_rx.recv_timeout(timeout) { + Ok(()) => {} + Err(_) => panic!("{msg}"), + } + worker.join().expect("worker panicked"); +} + +fn make_watched_picker(base: &std::path::Path) -> (SharedFilePicker, SharedFrecency) { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("Failed to create FilePicker"); + + assert!( + shared_picker.wait_for_scan(Duration::from_secs(10)), + "initial scan never completed" + ); + assert!( + shared_picker.wait_for_watcher(Duration::from_secs(10)), + "watcher never installed" + ); + + (shared_picker, shared_frecency) +} + +/// Hazard (1): debouncer event handler is waiting on `shared_picker.write()` +/// while the caller joins it from under the same guard. +#[test] +fn stop_background_monitor_under_write_lock_does_not_deadlock_file_events() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().to_path_buf(); + + for i in 0..4 { + fs::write(base.join(format!("file_{i}.txt")), format!("seed {i}\n")).unwrap(); + } + + let (shared_picker, _shared_frecency) = make_watched_picker(&base); + + // Produce enough filesystem churn that the debouncer has events + // queued and is likely mid-handler by the time we call stop. + for round in 0..8 { + for i in 0..4 { + let path = base.join(format!("file_{i}.txt")); + fs::write(&path, format!("edit {round}-{i}\n")).unwrap(); + } + } + + // Give the kernel time to deliver events into the debouncer queue + // (50 ms = default debouncer tick). + std::thread::sleep(Duration::from_millis(60)); + + let sp = shared_picker.clone(); + run_with_deadlock_guard( + "stop_background_monitor() deadlocked under shared_picker.write() — \ + the debouncer thread is likely waiting on the same write lock \ + while we join it", + Duration::from_secs(5), + move || { + let mut guard = sp.write().expect("write lock"); + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + }, + ); +} + +/// Hazard (2): owner thread holds the debouncer mutex while waiting +/// on `shared_picker.write()`, and `stop()` takes the debouncer mutex +/// under the caller's write guard. +#[test] +fn stop_background_monitor_under_write_lock_does_not_deadlock_new_dirs() { + let tmp = TempDir::new().unwrap(); + let base = tmp.path().to_path_buf(); + + fs::write(base.join("seed.txt"), "seed\n").unwrap(); + + let (shared_picker, _shared_frecency) = make_watched_picker(&base); + + // Create a burst of new subdirectories with files inside. On Linux + // the watcher event thread sends each new dir to `watch_tx`, and + // the owner thread processes them (taking the debouncer mutex + + // `shared_picker.write()`). On macOS the owner thread still runs + // `track_files_from_new_directories`, which takes the write lock. + for d in 0..8 { + let sub = base.join(format!("sub_{d}")); + fs::create_dir(&sub).unwrap(); + for f in 0..4 { + fs::write(sub.join(format!("f_{f}.txt")), format!("{d}-{f}\n")).unwrap(); + } + } + + std::thread::sleep(Duration::from_millis(120)); + + let sp = shared_picker.clone(); + run_with_deadlock_guard( + "stop_background_monitor() deadlocked under shared_picker.write() — \ + the watcher owner thread is likely holding the debouncer mutex and \ + waiting on the same write lock while we try to take the debouncer \ + mutex to tear it down", + Duration::from_secs(5), + move || { + let mut guard = sp.write().expect("write lock"); + if let Some(ref mut picker) = *guard { + picker.stop_background_monitor(); + } + }, + ); +} diff --git a/crates/fff-core/tests/watcher_thread_lifecycle_test.rs b/crates/fff-core/tests/watcher_thread_lifecycle_test.rs new file mode 100644 index 0000000..c7624f9 --- /dev/null +++ b/crates/fff-core/tests/watcher_thread_lifecycle_test.rs @@ -0,0 +1,197 @@ +#![cfg(target_os = "linux")] + +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use tempfile::TempDir; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency}; + +/// Thread comm names Linux exposes via `/proc/self/task/*/comm` are +/// capped at `TASK_COMM_LEN - 1 = 15` bytes. Our owner thread is named +/// `"fff-watcher-owner"` (17 bytes), so what actually appears in +/// `/proc` is the 15-byte truncation below. +const WATCHER_OWNER_THREAD_NAME: &str = "fff-watcher-own"; + +/// Walk `/proc/self/task/*/comm` and return how many live threads +/// carry `name` as their `comm`. +fn count_live_threads_named(name: &str) -> usize { + let Ok(dir) = fs::read_dir("/proc/self/task") else { + return 0; + }; + let mut count = 0usize; + for entry in dir.flatten() { + let comm_path = entry.path().join("comm"); + if let Ok(content) = fs::read_to_string(&comm_path) { + if content.trim_end() == name { + count += 1; + } + } + } + count +} + +/// Poll until the thread count matches `expected` or we hit `timeout`. +fn wait_for_thread_count(name: &str, expected: usize, timeout: Duration) -> usize { + let deadline = Instant::now() + timeout; + loop { + let count = count_live_threads_named(name); + if count == expected { + return count; + } + if Instant::now() >= deadline { + return count; + } + std::thread::sleep(Duration::from_millis(25)); + } +} + +fn seed_repo(base: &std::path::Path) { + fs::create_dir_all(base.join("src")).unwrap(); + fs::write(base.join("README.md"), "# seed\n").unwrap(); + fs::write(base.join("src/main.rs"), "fn main() {}\n").unwrap(); + fs::write(base.join("src/lib.rs"), "// lib\n").unwrap(); + + let _ = std::process::Command::new("git") + .args(["init", "-q", "-b", "main"]) + .current_dir(base) + .output(); +} + +fn spawn_watched_picker(base: PathBuf) -> (SharedFilePicker, SharedFrecency) { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().to_string(), + enable_mmap_cache: false, + enable_content_indexing: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("FilePicker::new_with_shared_state"); + + assert!( + shared_picker.wait_for_scan(Duration::from_secs(10)), + "initial scan did not complete" + ); + assert!( + shared_picker.wait_for_watcher(Duration::from_secs(10)), + "watcher did not install" + ); + + (shared_picker, shared_frecency) +} + +#[test] +fn watcher_threads_do_not_leak_across_picker_lifetimes() { + // this is needed because I run this within neovim with it's own fff owner thread lmao + let baseline = count_live_threads_named(WATCHER_OWNER_THREAD_NAME); + + const PICKER_COUNT: usize = 4; + + let mut tmpdirs: Vec = (0..PICKER_COUNT) + .map(|_| TempDir::new().expect("mktemp")) + .collect(); + for td in &tmpdirs { + seed_repo(td.path()); + } + + let mut pickers: Vec<(SharedFilePicker, SharedFrecency)> = tmpdirs + .iter() + .map(|td| spawn_watched_picker(td.path().canonicalize().expect("canonicalize tmp"))) + .collect(); + + let peak = wait_for_thread_count( + WATCHER_OWNER_THREAD_NAME, + baseline + PICKER_COUNT, + Duration::from_secs(5), + ); + assert_eq!( + peak, + baseline + PICKER_COUNT, + "expected {} watcher-owner threads alive (baseline {} + {} pickers), saw {}", + baseline + PICKER_COUNT, + baseline, + PICKER_COUNT, + peak, + ); + + for i in 0..PICKER_COUNT { + let expected_remaining = baseline + PICKER_COUNT - (i + 1); + let (sp, sf) = pickers.remove(0); + drop(sp); + drop(sf); + let count = wait_for_thread_count( + WATCHER_OWNER_THREAD_NAME, + expected_remaining, + Duration::from_secs(5), + ); + assert_eq!( + count, + expected_remaining, + "after dropping picker {}/{}: expected {} owner threads, saw {}", + i + 1, + PICKER_COUNT, + expected_remaining, + count, + ); + } + tmpdirs.clear(); + + let after_stage1 = count_live_threads_named(WATCHER_OWNER_THREAD_NAME); + assert_eq!( + after_stage1, baseline, + "stage 1 leaked watcher-owner threads: baseline {}, observed {}", + baseline, after_stage1, + ); + + const ROUNDS: usize = 3; + + for round in 0..ROUNDS { + let tmp = TempDir::new().expect("mktemp"); + seed_repo(tmp.path()); + let base = tmp.path().canonicalize().expect("canonicalize tmp"); + + let (sp, sf) = spawn_watched_picker(base); + + let during = wait_for_thread_count( + WATCHER_OWNER_THREAD_NAME, + baseline + 1, + Duration::from_secs(5), + ); + assert_eq!( + during, + baseline + 1, + "round {round}: expected 1 owner thread during run, saw {during} \ + (baseline {baseline})", + ); + + drop(sp); + drop(sf); + drop(tmp); + + let after = + wait_for_thread_count(WATCHER_OWNER_THREAD_NAME, baseline, Duration::from_secs(5)); + assert_eq!( + after, baseline, + "round {round}: owner thread leaked after teardown \ + (baseline {baseline}, observed {after})", + ); + } + + let final_count = count_live_threads_named(WATCHER_OWNER_THREAD_NAME); + assert_eq!( + final_count, baseline, + "watcher-owner threads leaked past the end of the test \ + (baseline {}, final {})", + baseline, final_count, + ); +} diff --git a/crates/fff-grep/Cargo.toml b/crates/fff-grep/Cargo.toml new file mode 100644 index 0000000..98aa8d4 --- /dev/null +++ b/crates/fff-grep/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "fff-grep" +description = "File grepping logic for fff" +license = "MIT" +authors = ["Dmitriy Kovalenko "] +version = "0.9.6" +edition = "2024" + +[dependencies] +bstr = { version = "1.6.2", default-features = false, features = ["std"] } +memchr = "2.6.3" diff --git a/crates/fff-grep/src/lib.rs b/crates/fff-grep/src/lib.rs new file mode 100644 index 0000000..53e7ae5 --- /dev/null +++ b/crates/fff-grep/src/lib.rs @@ -0,0 +1,19 @@ +/*! +Simplified grep-searcher for fff.nvim. + +Provides line-oriented search over byte slices with optional multi-line support. +Only `search_slice` is supported -- no file/reader/mmap search. +*/ + +#![deny(missing_docs)] + +pub use crate::{ + matcher::{LineTerminator, Match, Matcher, NoError}, + searcher::{Searcher, SearcherBuilder}, + sink::{Sink, SinkError, SinkFinish, SinkMatch}, +}; + +pub mod lines; +pub mod matcher; +mod searcher; +mod sink; diff --git a/crates/fff-grep/src/lines.rs b/crates/fff-grep/src/lines.rs new file mode 100644 index 0000000..0d7fabf --- /dev/null +++ b/crates/fff-grep/src/lines.rs @@ -0,0 +1,232 @@ +/*! +A collection of routines for performing operations on lines. +*/ + +use bstr::ByteSlice; + +use crate::matcher::{LineTerminator, Match}; + +/// An explicit iterator over lines in a particular slice of bytes. +/// +/// This iterator avoids borrowing the bytes themselves, and instead requires +/// callers to explicitly provide the bytes when moving through the iterator. +/// +/// Line terminators are considered part of the line they terminate. All lines +/// yielded by the iterator are guaranteed to be non-empty. +#[derive(Debug)] +pub struct LineStep { + line_term: u8, + pos: usize, + end: usize, +} + +impl LineStep { + /// Create a new line iterator over the given range of bytes using the + /// given line terminator. + pub fn new(line_term: u8, start: usize, end: usize) -> LineStep { + LineStep { + line_term, + pos: start, + end, + } + } + + /// Like next, but returns a `Match` instead of a tuple. + #[inline(always)] + pub fn next_match(&mut self, bytes: &[u8]) -> Option { + self.next_impl(bytes).map(|(s, e)| Match::new(s, e)) + } + + #[inline(always)] + fn next_impl(&mut self, mut bytes: &[u8]) -> Option<(usize, usize)> { + bytes = &bytes[..self.end]; + match bytes[self.pos..].find_byte(self.line_term) { + None => { + if self.pos < bytes.len() { + let m = (self.pos, bytes.len()); + assert!(m.0 <= m.1); + + self.pos = m.1; + Some(m) + } else { + None + } + } + Some(line_end) => { + let m = (self.pos, self.pos + line_end + 1); + assert!(m.0 <= m.1); + + self.pos = m.1; + Some(m) + } + } + } +} + +/// Count the number of occurrences of `line_term` in `bytes`. +pub fn count(bytes: &[u8], line_term: u8) -> u64 { + memchr::memchr_iter(line_term, bytes).count() as u64 +} + +/// Given a line that possibly ends with a terminator, return that line without the terminator. +#[inline(always)] +pub fn without_terminator(bytes: &[u8], line_term: LineTerminator) -> &[u8] { + let line_term = line_term.as_bytes(); + let start = bytes.len().saturating_sub(line_term.len()); + if bytes.get(start..) == Some(line_term) { + return &bytes[..bytes.len() - line_term.len()]; + } + bytes +} + +/// Return the start and end offsets of the lines containing the given range +/// of bytes. +/// +/// Line terminators are considered part of the line they terminate. +#[inline(always)] +pub fn locate(bytes: &[u8], line_term: u8, range: Match) -> Match { + let line_start = bytes[..range.start()] + .rfind_byte(line_term) + .map_or(0, |i| i + 1); + let line_end = if range.end() > line_start && bytes[range.end() - 1] == line_term { + range.end() + } else { + bytes[range.end()..] + .find_byte(line_term) + .map_or(bytes.len(), |i| range.end() + i + 1) + }; + Match::new(line_start, line_end) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SHERLOCK: &str = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +be, to a very large extent, the result of luck. Sherlock Holmes +can extract a clew from a wisp of straw or a flake of cigar ash; +but Doctor Watson has to have it taken out for him and dusted, +and exhibited clearly, with a label attached.\ +"; + + fn m(start: usize, end: usize) -> Match { + Match::new(start, end) + } + + fn lines(text: &str) -> Vec<&str> { + let mut results = vec![]; + let mut it = LineStep::new(b'\n', 0, text.len()); + while let Some(m) = it.next_match(text.as_bytes()) { + results.push(&text[m]); + } + results + } + + fn line_ranges(text: &str) -> Vec> { + let mut results = vec![]; + let mut it = LineStep::new(b'\n', 0, text.len()); + while let Some(m) = it.next_match(text.as_bytes()) { + results.push(m.start()..m.end()); + } + results + } + + fn loc(text: &str, start: usize, end: usize) -> Match { + locate(text.as_bytes(), b'\n', Match::new(start, end)) + } + + #[test] + fn line_count() { + assert_eq!(0, count(b"", b'\n')); + assert_eq!(1, count(b"\n", b'\n')); + assert_eq!(2, count(b"\n\n", b'\n')); + assert_eq!(2, count(b"a\nb\nc", b'\n')); + } + + #[test] + fn line_locate() { + let t = SHERLOCK; + let lines = line_ranges(t); + + assert_eq!( + loc(t, lines[0].start, lines[0].end), + m(lines[0].start, lines[0].end) + ); + assert_eq!( + loc(t, lines[0].start + 1, lines[0].end), + m(lines[0].start, lines[0].end) + ); + assert_eq!( + loc(t, lines[0].end - 1, lines[0].end), + m(lines[0].start, lines[0].end) + ); + assert_eq!( + loc(t, lines[0].end, lines[0].end), + m(lines[1].start, lines[1].end) + ); + + assert_eq!( + loc(t, lines[5].start, lines[5].end), + m(lines[5].start, lines[5].end) + ); + assert_eq!( + loc(t, lines[5].start + 1, lines[5].end), + m(lines[5].start, lines[5].end) + ); + assert_eq!( + loc(t, lines[5].end - 1, lines[5].end), + m(lines[5].start, lines[5].end) + ); + assert_eq!( + loc(t, lines[5].end, lines[5].end), + m(lines[5].start, lines[5].end) + ); + } + + #[test] + fn line_locate_weird() { + assert_eq!(loc("", 0, 0), m(0, 0)); + + assert_eq!(loc("\n", 0, 1), m(0, 1)); + assert_eq!(loc("\n", 1, 1), m(1, 1)); + + assert_eq!(loc("\n\n", 0, 0), m(0, 1)); + assert_eq!(loc("\n\n", 0, 1), m(0, 1)); + assert_eq!(loc("\n\n", 1, 1), m(1, 2)); + assert_eq!(loc("\n\n", 1, 2), m(1, 2)); + assert_eq!(loc("\n\n", 2, 2), m(2, 2)); + + assert_eq!(loc("a\nb\nc", 0, 1), m(0, 2)); + assert_eq!(loc("a\nb\nc", 1, 2), m(0, 2)); + assert_eq!(loc("a\nb\nc", 2, 3), m(2, 4)); + assert_eq!(loc("a\nb\nc", 3, 4), m(2, 4)); + assert_eq!(loc("a\nb\nc", 4, 5), m(4, 5)); + assert_eq!(loc("a\nb\nc", 5, 5), m(4, 5)); + } + + #[test] + fn line_iter() { + assert_eq!(lines("abc"), vec!["abc"]); + + assert_eq!(lines("abc\n"), vec!["abc\n"]); + assert_eq!(lines("abc\nxyz"), vec!["abc\n", "xyz"]); + assert_eq!(lines("abc\nxyz\n"), vec!["abc\n", "xyz\n"]); + + assert_eq!(lines("abc\n\n"), vec!["abc\n", "\n"]); + assert_eq!(lines("abc\n\n\n"), vec!["abc\n", "\n", "\n"]); + assert_eq!(lines("abc\n\nxyz"), vec!["abc\n", "\n", "xyz"]); + assert_eq!(lines("abc\n\nxyz\n"), vec!["abc\n", "\n", "xyz\n"]); + assert_eq!(lines("abc\nxyz\n\n"), vec!["abc\n", "xyz\n", "\n"]); + + assert_eq!(lines("\n"), vec!["\n"]); + assert_eq!(lines(""), Vec::<&str>::new()); + } + + #[test] + fn line_iter_empty() { + let mut it = LineStep::new(b'\n', 0, 0); + assert_eq!(it.next_match(b"abc"), None); + } +} diff --git a/crates/fff-grep/src/matcher.rs b/crates/fff-grep/src/matcher.rs new file mode 100644 index 0000000..60fa1db --- /dev/null +++ b/crates/fff-grep/src/matcher.rs @@ -0,0 +1,175 @@ +//! Matcher trait inspired by ripgrep's `Matcher` just simpler + +/// A byte range representing a match. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Match { + start: usize, + end: usize, +} + +impl Match { + /// Create a new match from start/end byte offsets. + #[inline] + pub fn new(start: usize, end: usize) -> Match { + debug_assert!(start <= end); + Match { start, end } + } + + /// Create a zero-width match at `offset`. + #[inline] + pub fn zero(offset: usize) -> Match { + Match { + start: offset, + end: offset, + } + } + + /// Start byte offset. + #[inline] + pub fn start(&self) -> usize { + self.start + } + + /// End byte offset (exclusive). + #[inline] + pub fn end(&self) -> usize { + self.end + } + + /// Return a copy with a different end offset. + #[inline] + pub fn with_end(&self, end: usize) -> Match { + debug_assert!(self.start <= end); + Match { end, ..*self } + } + + /// Shift both offsets forward by `amount`. + #[inline] + pub fn offset(&self, amount: usize) -> Match { + Match { + start: self.start + amount, + end: self.end + amount, + } + } + + /// Byte length of the match. + #[inline] + pub fn len(&self) -> usize { + self.end - self.start + } + + /// True if this is a zero-width match. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl std::ops::Index for [u8] { + type Output = [u8]; + + #[inline] + fn index(&self, index: Match) -> &[u8] { + &self[index.start..index.end] + } +} + +impl std::ops::IndexMut for [u8] { + #[inline] + fn index_mut(&mut self, index: Match) -> &mut [u8] { + &mut self[index.start..index.end] + } +} + +impl std::ops::Index for str { + type Output = str; + + #[inline] + fn index(&self, index: Match) -> &str { + &self[index.start..index.end] + } +} + +/// A line terminator (always a single byte for fff — no CRLF support needed). +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct LineTerminator(u8); + +impl LineTerminator { + /// Create a line terminator from a single byte. + #[inline] + pub fn byte(byte: u8) -> LineTerminator { + LineTerminator(byte) + } + + /// Return the terminator byte. + #[inline] + pub fn as_byte(&self) -> u8 { + self.0 + } + + /// Return the terminator as a single-element byte slice. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + std::slice::from_ref(&self.0) + } +} + +impl Default for LineTerminator { + #[inline] + fn default() -> LineTerminator { + LineTerminator(b'\n') + } +} + +/// An error type for matchers that never produce errors. +#[derive(Debug, Eq, PartialEq)] +pub struct NoError(()); + +impl std::error::Error for NoError {} + +impl std::fmt::Display for NoError { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + unreachable!("NoError should never be instantiated") + } +} + +/// A matcher finds byte-level matches in a haystack. +pub trait Matcher { + /// The error type (use [`NoError`] for infallible matchers). + type Error: std::fmt::Display; + + /// Find the first match at or after `at` in `haystack`. + fn find_at(&self, haystack: &[u8], at: usize) -> Result, Self::Error>; + + /// Find the first match in `haystack`. + #[inline] + fn find(&self, haystack: &[u8]) -> Result, Self::Error> { + self.find_at(haystack, 0) + } + + /// The line terminator this matcher guarantees will never appear in a match. + /// Return `None` if the matcher can match across lines. + #[inline] + fn line_terminator(&self) -> Option { + None + } +} + +impl Matcher for &M { + type Error = M::Error; + + #[inline] + fn find_at(&self, haystack: &[u8], at: usize) -> Result, Self::Error> { + (*self).find_at(haystack, at) + } + + #[inline] + fn find(&self, haystack: &[u8]) -> Result, Self::Error> { + (*self).find(haystack) + } + + #[inline] + fn line_terminator(&self) -> Option { + (*self).line_terminator() + } +} diff --git a/crates/fff-grep/src/searcher/core.rs b/crates/fff-grep/src/searcher/core.rs new file mode 100644 index 0000000..c182ff0 --- /dev/null +++ b/crates/fff-grep/src/searcher/core.rs @@ -0,0 +1,139 @@ +use crate::{ + lines, + matcher::Matcher, + searcher::{Config, Range, Searcher}, + sink::{Sink, SinkError, SinkFinish, SinkMatch}, +}; + +#[derive(Debug)] +pub(crate) struct Core<'s, M: 's, S> { + config: &'s Config, + matcher: M, + searcher: &'s Searcher, + sink: S, + pos: usize, + absolute_byte_offset: u64, + line_number: Option, + last_line_counted: usize, + last_line_visited: usize, +} + +impl<'s, M: Matcher, S: Sink> Core<'s, M, S> { + pub(crate) fn new(searcher: &'s Searcher, matcher: M, sink: S) -> Core<'s, M, S> { + let line_number = if searcher.config.line_number { + Some(1) + } else { + None + }; + Core { + config: &searcher.config, + matcher, + searcher, + sink, + pos: 0, + absolute_byte_offset: 0, + line_number, + last_line_counted: 0, + last_line_visited: 0, + } + } + + pub(crate) fn pos(&self) -> usize { + self.pos + } + + pub(crate) fn set_pos(&mut self, pos: usize) { + self.pos = pos; + } + + pub(crate) fn matched(&mut self, buf: &[u8], range: &Range) -> Result { + self.sink_matched(buf, range) + } + + pub(crate) fn find(&mut self, slice: &[u8]) -> Result, S::Error> { + self.matcher.find(slice).map_err(S::Error::error_message) + } + + pub(crate) fn begin(&mut self) -> Result { + self.sink.begin(self.searcher) + } + + pub(crate) fn finish(&mut self, byte_count: u64) -> Result<(), S::Error> { + self.sink.finish(self.searcher, &SinkFinish { byte_count }) + } + + pub(crate) fn match_by_line(&mut self, buf: &[u8]) -> Result { + while !buf[self.pos()..].is_empty() { + if let Some(line) = self.find_by_line(buf)? { + self.set_pos(line.end()); + if !self.sink_matched(buf, &line)? { + return Ok(false); + } + } else { + break; + } + } + self.set_pos(buf.len()); + Ok(true) + } + + #[inline(always)] + fn find_by_line(&mut self, buf: &[u8]) -> Result, S::Error> { + let mut pos = self.pos(); + while !buf[pos..].is_empty() { + let mat = match self + .matcher + .find(&buf[pos..]) + .map_err(S::Error::error_message)? + { + None => return Ok(None), + Some(m) => m, + }; + let line = lines::locate( + buf, + self.config.line_term.as_byte(), + Range::zero(mat.start()).offset(pos), + ); + if line.start() == buf.len() { + pos = buf.len(); + continue; + } + return Ok(Some(line)); + } + Ok(None) + } + + #[inline(always)] + fn sink_matched(&mut self, buf: &[u8], range: &Range) -> Result { + self.count_lines(buf, range.start()); + let offset = self.absolute_byte_offset + range.start() as u64; + let linebuf = &buf[*range]; + let keepgoing = self.sink.matched( + self.searcher, + &SinkMatch { + bytes: linebuf, + absolute_byte_offset: offset, + line_number: self.line_number, + buffer: buf, + bytes_range_in_buffer: range.start()..range.end(), + }, + )?; + if !keepgoing { + return Ok(false); + } + self.last_line_visited = range.end(); + Ok(true) + } + + fn count_lines(&mut self, buf: &[u8], upto: usize) { + if let Some(ref mut line_number) = self.line_number { + if self.last_line_counted >= upto { + return; + } + let slice = &buf[self.last_line_counted..upto]; + let count = lines::count(slice, self.config.line_term.as_byte()); + *line_number += count; + self.last_line_counted = upto; + } + } +} diff --git a/crates/fff-grep/src/searcher/glue.rs b/crates/fff-grep/src/searcher/glue.rs new file mode 100644 index 0000000..fbdc643 --- /dev/null +++ b/crates/fff-grep/src/searcher/glue.rs @@ -0,0 +1,127 @@ +use crate::{ + lines, + matcher::Matcher, + searcher::{Config, Range, Searcher, core::Core}, + sink::Sink, +}; + +#[derive(Debug)] +pub(crate) struct SliceByLine<'s, M, S> { + core: Core<'s, M, S>, + slice: &'s [u8], +} + +impl<'s, M: Matcher, S: Sink> SliceByLine<'s, M, S> { + pub(crate) fn new( + searcher: &'s Searcher, + matcher: M, + slice: &'s [u8], + write_to: S, + ) -> SliceByLine<'s, M, S> { + debug_assert!(!searcher.multi_line_with_matcher(&matcher)); + + SliceByLine { + core: Core::new(searcher, matcher, write_to), + slice, + } + } + + pub(crate) fn run(mut self) -> Result<(), S::Error> { + if self.core.begin()? { + while !self.slice[self.core.pos()..].is_empty() + && self.core.match_by_line(self.slice)? + {} + } + let byte_count = self.slice.len() as u64; + self.core.finish(byte_count) + } +} + +#[derive(Debug)] +pub(crate) struct MultiLine<'s, M, S> { + config: &'s Config, + core: Core<'s, M, S>, + slice: &'s [u8], + last_match: Option, +} + +impl<'s, M: Matcher, S: Sink> MultiLine<'s, M, S> { + pub(crate) fn new( + searcher: &'s Searcher, + matcher: M, + slice: &'s [u8], + write_to: S, + ) -> MultiLine<'s, M, S> { + debug_assert!(searcher.multi_line_with_matcher(&matcher)); + + MultiLine { + config: &searcher.config, + core: Core::new(searcher, matcher, write_to), + slice, + last_match: None, + } + } + + pub(crate) fn run(mut self) -> Result<(), S::Error> { + if self.core.begin()? { + let mut keepgoing = true; + while !self.slice[self.core.pos()..].is_empty() && keepgoing { + keepgoing = self.sink()?; + } + if keepgoing && let Some(last_match) = self.last_match.take() { + self.sink_matched(&last_match)?; + } + } + let byte_count = self.slice.len() as u64; + self.core.finish(byte_count) + } + + fn sink(&mut self) -> Result { + let mat = match self.find()? { + Some(range) => range, + None => { + self.core.set_pos(self.slice.len()); + return Ok(true); + } + }; + self.advance(&mat); + + let line = lines::locate(self.slice, self.config.line_term.as_byte(), mat); + match self.last_match.take() { + None => { + self.last_match = Some(line); + Ok(true) + } + Some(last_match) => { + if last_match.end() >= line.start() { + self.last_match = Some(last_match.with_end(line.end())); + Ok(true) + } else { + self.last_match = Some(line); + self.sink_matched(&last_match) + } + } + } + } + + fn sink_matched(&mut self, range: &Range) -> Result { + if range.is_empty() { + return Ok(false); + } + self.core.matched(self.slice, range) + } + + fn find(&mut self) -> Result, S::Error> { + self.core + .find(&self.slice[self.core.pos()..]) + .map(|m| m.map(|m| m.offset(self.core.pos()))) + } + + fn advance(&mut self, range: &Range) { + self.core.set_pos(range.end()); + if range.is_empty() && self.core.pos() < self.slice.len() { + let newpos = self.core.pos() + 1; + self.core.set_pos(newpos); + } + } +} diff --git a/crates/fff-grep/src/searcher/mod.rs b/crates/fff-grep/src/searcher/mod.rs new file mode 100644 index 0000000..b8aa006 --- /dev/null +++ b/crates/fff-grep/src/searcher/mod.rs @@ -0,0 +1,194 @@ +use crate::{ + matcher::{LineTerminator, Match, Matcher}, + searcher::glue::{MultiLine, SliceByLine}, + sink::{Sink, SinkError}, +}; + +mod core; +mod glue; + +/// We use this type alias since we want the ergonomics of a matcher's `Match` +/// type, but in practice, we use it for arbitrary ranges, so give it a more +/// accurate name. This is only used in the searcher's internals. +type Range = Match; + +/// An error that can occur when building a searcher. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub(crate) enum ConfigError { + /// Occurs when a matcher reports a line terminator that is different than + /// the one configured in the searcher. + MismatchedLineTerminators { + /// The matcher's line terminator. + matcher: LineTerminator, + /// The searcher's line terminator. + searcher: LineTerminator, + }, +} + +impl std::error::Error for ConfigError {} + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + ConfigError::MismatchedLineTerminators { matcher, searcher } => { + write!( + f, + "grep config error: mismatched line terminators, \ + matcher has {:?} but searcher has {:?}", + matcher, searcher + ) + } + } + } +} + +/// The internal configuration of a searcher. +#[derive(Clone, Debug)] +pub(crate) struct Config { + /// The line terminator to use. + pub(crate) line_term: LineTerminator, + /// Whether to count line numbers. + pub(crate) line_number: bool, + /// Whether to enable matching across multiple lines. + multi_line: bool, +} + +impl Default for Config { + fn default() -> Config { + Config { + line_term: LineTerminator::default(), + line_number: true, + multi_line: false, + } + } +} + +/// A builder for configuring a searcher. +#[derive(Clone, Debug)] +pub struct SearcherBuilder { + config: Config, +} + +impl Default for SearcherBuilder { + fn default() -> SearcherBuilder { + SearcherBuilder::new() + } +} + +impl SearcherBuilder { + /// Create a new searcher builder with a default configuration. + pub fn new() -> SearcherBuilder { + SearcherBuilder { + config: Config::default(), + } + } + + /// Build a searcher. + pub fn build(&self) -> Searcher { + Searcher { + config: self.config.clone(), + } + } + + /// Whether to count and include line numbers with matching lines. + pub fn line_number(&mut self, yes: bool) -> &mut SearcherBuilder { + self.config.line_number = yes; + self + } + + /// Whether to enable multi line search or not. + pub fn multi_line(&mut self, yes: bool) -> &mut SearcherBuilder { + self.config.multi_line = yes; + self + } +} + +/// A searcher executes searches over a haystack and writes results to a caller +/// provided sink. +#[derive(Clone, Debug)] +pub struct Searcher { + pub(crate) config: Config, +} + +impl Searcher { + /// Create a new searcher with a default configuration. + pub fn new() -> Searcher { + SearcherBuilder::new().build() + } + + /// Execute a search over the given slice and write the results to the + /// given sink. + pub fn search_slice(&self, matcher: M, slice: &[u8], write_to: S) -> Result<(), S::Error> + where + M: Matcher, + S: Sink, + { + self.check_config(&matcher) + .map_err(S::Error::error_message)?; + + if self.multi_line_with_matcher(&matcher) { + MultiLine::new(self, matcher, slice, write_to).run() + } else { + SliceByLine::new(self, matcher, slice, write_to).run() + } + } + + /// Check that the searcher's configuration and the matcher are consistent. + fn check_config(&self, matcher: M) -> Result<(), ConfigError> { + let matcher_line_term = match matcher.line_terminator() { + None => return Ok(()), + Some(line_term) => line_term, + }; + if matcher_line_term != self.config.line_term { + return Err(ConfigError::MismatchedLineTerminators { + matcher: matcher_line_term, + searcher: self.config.line_term, + }); + } + Ok(()) + } +} + +impl Default for Searcher { + fn default() -> Self { + Self::new() + } +} + +/// Configuration query methods used by the sink and internal search core. +impl Searcher { + /// Returns the line terminator used by this searcher. + #[inline] + pub fn line_terminator(&self) -> LineTerminator { + self.config.line_term + } + + /// Returns true if and only if this searcher is configured to count line + /// numbers. + #[inline] + pub fn line_number(&self) -> bool { + self.config.line_number + } + + /// Returns true if and only if this searcher is configured to perform + /// multi line search. + #[inline] + pub fn multi_line(&self) -> bool { + self.config.multi_line + } + + /// Returns true if and only if this searcher will choose a multi-line + /// strategy given the provided matcher. + pub fn multi_line_with_matcher(&self, matcher: M) -> bool { + if !self.multi_line() { + return false; + } + if let Some(line_term) = matcher.line_terminator() + && line_term == self.line_terminator() + { + return false; + } + true + } +} diff --git a/crates/fff-grep/src/sink.rs b/crates/fff-grep/src/sink.rs new file mode 100644 index 0000000..ba0e29f --- /dev/null +++ b/crates/fff-grep/src/sink.rs @@ -0,0 +1,138 @@ +use std::io; + +use crate::searcher::Searcher; + +/// A trait that describes errors that can be reported by searchers and +/// implementations of `Sink`. +pub trait SinkError: Sized { + /// A constructor for converting any value that satisfies the + /// `std::fmt::Display` trait into an error. + fn error_message(message: T) -> Self; + + /// A constructor for converting I/O errors that occur while searching into + /// an error of this type. + fn error_io(err: io::Error) -> Self { + Self::error_message(err) + } +} + +impl SinkError for io::Error { + fn error_message(message: T) -> io::Error { + io::Error::other(message.to_string()) + } + + fn error_io(err: io::Error) -> io::Error { + err + } +} + +/// A trait that defines how results from searchers are handled. +/// +/// The searcher follows the "push" model: the searcher drives execution and +/// pushes results back to the caller via this trait. +pub trait Sink { + /// The type of an error that should be reported by a searcher. + type Error: SinkError; + + /// This method is called whenever a match is found. + /// + /// If this returns `true`, then searching continues. If this returns + /// `false`, then searching is stopped immediately and `finish` is called. + fn matched(&mut self, _searcher: &Searcher, _mat: &SinkMatch<'_>) -> Result; + + /// This method is called when a search has begun, before any search is + /// executed. By default, this does nothing. + #[inline] + fn begin(&mut self, _searcher: &Searcher) -> Result { + Ok(true) + } + + /// This method is called when a search has completed. By default, this + /// does nothing. + #[inline] + fn finish(&mut self, _searcher: &Searcher, _: &SinkFinish) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl Sink for &mut S { + type Error = S::Error; + + #[inline] + fn matched(&mut self, searcher: &Searcher, mat: &SinkMatch<'_>) -> Result { + (**self).matched(searcher, mat) + } + + #[inline] + fn begin(&mut self, searcher: &Searcher) -> Result { + (**self).begin(searcher) + } + + #[inline] + fn finish(&mut self, searcher: &Searcher, sink_finish: &SinkFinish) -> Result<(), S::Error> { + (**self).finish(searcher, sink_finish) + } +} + +/// Summary data reported at the end of a search. +#[derive(Clone, Debug)] +pub struct SinkFinish { + pub(crate) byte_count: u64, +} + +impl SinkFinish { + /// Return the total number of bytes searched. + #[inline] + pub fn byte_count(&self) -> u64 { + self.byte_count + } +} + +/// A type that describes a match reported by a searcher. +#[derive(Clone, Debug)] +pub struct SinkMatch<'b> { + pub(crate) bytes: &'b [u8], + pub(crate) absolute_byte_offset: u64, + pub(crate) line_number: Option, + pub(crate) buffer: &'b [u8], + pub(crate) bytes_range_in_buffer: std::ops::Range, +} + +impl<'b> SinkMatch<'b> { + /// Returns the bytes for all matching lines, including the line + /// terminators, if they exist. + #[inline] + pub fn bytes(&self) -> &'b [u8] { + self.bytes + } + + /// Returns the absolute byte offset of the start of this match. This + /// offset is absolute in that it is relative to the very beginning of the + /// input in a search. + #[inline] + pub fn absolute_byte_offset(&self) -> u64 { + self.absolute_byte_offset + } + + /// Returns the line number of the first line in this match, if available. + /// + /// Line numbers are only available when the search builder is instructed + /// to compute them. + #[inline] + pub fn line_number(&self) -> Option { + self.line_number + } + + /// Exposes as much of the underlying buffer that was searched as possible. + #[inline] + pub fn buffer(&self) -> &'b [u8] { + self.buffer + } + + /// Returns a range that corresponds to where [`SinkMatch::bytes`] appears + /// in [`SinkMatch::buffer`]. + #[inline] + pub fn bytes_range_in_buffer(&self) -> std::ops::Range { + self.bytes_range_in_buffer.clone() + } +} diff --git a/crates/fff-mcp/Cargo.toml b/crates/fff-mcp/Cargo.toml new file mode 100644 index 0000000..9f46651 --- /dev/null +++ b/crates/fff-mcp/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "fff-mcp" +version = "0.9.6" +edition = "2024" +description = "MCP server for FFF file finder - drop-in replacement for AI code assistant search tools" +license = "MIT" + +[[bin]] +name = "fff-mcp" +path = "src/main.rs" + +[features] +# Pure-Rust walker by default; opt into zlob explicitly (needs Zig). +default = ["ripgrep"] +ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"] +zlob = ["fff/zlob", "fff-query-parser/zlob"] + +[dependencies] +fff = { package = "fff-search", path = "../fff-core", default-features = false , version = "0.9.6", features = ["definitions"] } +fff-query-parser = { path = "../fff-query-parser", default-features = false , version = "0.9.6" } +mimalloc = { workspace = true } +rmcp = { version = "1.7.0", features = ["server", "transport-io"] } +schemars = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1", features = ["full"] } +tracing = { workspace = true } +git2 = { workspace = true } +clap = { version = "4", features = ["derive", "env"] } diff --git a/crates/fff-mcp/build.rs b/crates/fff-mcp/build.rs new file mode 100644 index 0000000..30a1d0e --- /dev/null +++ b/crates/fff-mcp/build.rs @@ -0,0 +1,15 @@ +fn main() { + // Embed the git commit hash at build time for update checking. + let hash = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + println!("cargo:rustc-env=FFF_GIT_HASH={}", hash); + println!("cargo:rerun-if-changed=../../.git/HEAD"); + println!("cargo:rerun-if-changed=../../.git/refs/"); +} diff --git a/crates/fff-mcp/server/.gitkeep b/crates/fff-mcp/server/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/crates/fff-mcp/src/cursor.rs b/crates/fff-mcp/src/cursor.rs new file mode 100644 index 0000000..fc5ad65 --- /dev/null +++ b/crates/fff-mcp/src/cursor.rs @@ -0,0 +1,52 @@ +//! Cursor store for grep pagination. +//! +//! Maintains an in-memory map of opaque cursor IDs to file offsets. +//! Cursors are evicted LRU-style when the store exceeds capacity. + +use std::collections::{HashMap, VecDeque}; + +const MAX_CURSORS: usize = 20; + +/// Stores cursor state for paginated grep results. +pub struct CursorStore { + counter: u64, + /// Map from cursor ID string → file offset for next page. + cursors: HashMap, + /// Insertion order for LRU eviction. + insertion_order: VecDeque, +} + +impl CursorStore { + pub fn new() -> Self { + Self { + counter: 0, + cursors: HashMap::new(), + insertion_order: VecDeque::new(), + } + } + + /// Store a cursor and return its opaque ID string. + pub fn store(&mut self, file_offset: usize) -> String { + self.counter = self.counter.wrapping_add(1); + let id = self.counter.to_string(); + + self.cursors.insert(id.clone(), file_offset); + self.insertion_order.push_back(id.clone()); + + // Evict oldest cursors + while self.cursors.len() > MAX_CURSORS { + if let Some(oldest) = self.insertion_order.pop_front() { + self.cursors.remove(&oldest); + } else { + break; + } + } + + id + } + + /// Retrieve the file offset for a cursor ID. + pub fn get(&self, id: &str) -> Option { + self.cursors.get(id).copied() + } +} diff --git a/crates/fff-mcp/src/healthcheck.rs b/crates/fff-mcp/src/healthcheck.rs new file mode 100644 index 0000000..cc7db9e --- /dev/null +++ b/crates/fff-mcp/src/healthcheck.rs @@ -0,0 +1,117 @@ +use crate::Args; +use git2::Repository; + +fn check(label: &str, ok: bool, detail: &str) -> bool { + let marker = if ok { "+" } else { "x" }; + println!(" [{marker}] {label}: {detail}"); + ok +} + +fn warn(label: &str, detail: &str) { + println!(" [!] {label}: {detail}"); +} + +pub fn run_healthcheck(args: &Args) -> Result<(), Box> { + let version = concat!(env!("CARGO_PKG_VERSION"), " (", env!("FFF_GIT_HASH"), ")"); + println!("fff-mcp {version}\n"); + + let mut all_ok = true; + + // 1. Base path + let base_path = args.base_path.clone().unwrap_or_else(|| { + std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .to_string() + }); + + let path_exists = std::path::Path::new(&base_path).is_dir(); + all_ok &= check( + "Base path", + path_exists, + if path_exists { + &base_path + } else { + "directory does not exist" + }, + ); + + // 2. Git repository + match Repository::discover(&base_path) { + Ok(repo) => { + if let Some(workdir) = repo.workdir() { + all_ok &= check("Git repository", true, &format!("{}", workdir.display())); + } else { + all_ok &= check("Git repository", true, "bare repository"); + } + } + Err(_) => { + // Not fatal — fff-mcp works without git, but worth flagging. + warn( + "Git repository", + "not found (fff-mcp will still work, but git-status features are disabled)", + ); + } + } + + // 3. Frecency database + if let Some(ref db_path) = args.frecency_db_path { + let parent_ok = std::path::Path::new(db_path) + .parent() + .is_some_and(|p| p.is_dir()); + all_ok &= check( + "Frecency DB", + parent_ok, + if parent_ok { + db_path + } else { + "parent directory does not exist" + }, + ); + } else { + check("Frecency DB", false, "path not resolved"); + } + + // 4. Query history database + if let Some(ref db_path) = args.history_db_path { + let parent_ok = std::path::Path::new(db_path) + .parent() + .is_some_and(|p| p.is_dir()); + all_ok &= check( + "History DB", + parent_ok, + if parent_ok { + db_path + } else { + "parent directory does not exist" + }, + ); + } else { + check("History DB", false, "path not resolved"); + } + + // 5. Log path hint (per-session files written next to this path) + if let Some(ref log_path) = args.log_file { + let parent_ok = std::path::Path::new(log_path) + .parent() + .is_some_and(|p| p.is_dir() || p.parent().is_some()); + all_ok &= check( + "Log path", + parent_ok, + if parent_ok { + log_path + } else { + "parent directory does not exist" + }, + ); + } else { + check("Log path", false, "path not resolved"); + } + + if all_ok { + println!("All checks passed."); + Ok(()) + } else { + Err("Some checks failed — review the items marked [x] above.".into()) + } +} diff --git a/crates/fff-mcp/src/main.rs b/crates/fff-mcp/src/main.rs new file mode 100644 index 0000000..1777282 --- /dev/null +++ b/crates/fff-mcp/src/main.rs @@ -0,0 +1,372 @@ +mod cursor; +mod healthcheck; +mod output; +mod server; +mod update_check; + +use clap::Parser; +use fff::file_picker::FilePicker; +use fff::frecency::FrecencyTracker; +use fff::{FFFMode, SharedFilePicker, SharedFrecency}; +use git2::Repository; +use mimalloc::MiMalloc; +use rmcp::{ServiceExt, transport::stdio}; +use server::FffServer; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +pub const MCP_INSTRUCTIONS: &str = concat!( + "FFF is a fast file finder with frecency-ranked results (frequent/recent files first, git-dirty files boosted).\n", + "\n", + "## Which Tool Should I Use?\n", + "\n", + "- **grep**: DEFAULT tool. Searches file CONTENTS -- definitions, usage, patterns. Use when you have a specific name or pattern.\n", + "- **find_files**: Explores which files/modules exist for a topic. Use when you DON'T have a specific identifier or LOOKING FOR A FILE.\n", + "- **multi_grep**: OR logic across multiple patterns. Use for case variants (e.g. ['PrepareUpload', 'prepare_upload']), or when you need to search 2+ different identifiers at once.\n", + "\n", + "## Core Rules\n", + "\n", + "### 1. Search BARE IDENTIFIERS only\n", + "Grep matches single lines. Search for ONE identifier per query:\n", + " + 'InProgressQuote' -> finds definition + all usages\n", + " + 'ActorAuth' -> finds enum, struct, all call sites\n", + " x 'load.*metadata.*InProgressQuote' -> regex spanning multiple tokens, 0 results\n", + " x 'ctx.data::' -> code syntax, too specific, 0 results\n", + " x 'struct ActorAuth' -> adding keywords narrows results, misses enums/traits/type aliases\n", + " x 'TODO.*#\\d+' -> complex regex, use simple 'TODO' then filter visually\n", + "\n", + "### 2. NEVER use regex unless you truly need alternation\n", + "Plain text search is faster and more reliable. Regex patterns like `.*`, `\\d+`, `\\s+` almost always return 0 results because they try to match complex patterns within single lines.\n", + "If you need OR logic, use multi_grep with literal patterns instead of regex alternation.\n", + "\n", + "### 3. Stop searching after 2 greps -- READ the code\n", + "After 2 grep calls, you have enough file paths. Read the top result to understand the code.\n", + "Do NOT keep grepping with variations. More greps != better understanding.\n", + "\n", + "### 4. Use multi_grep for multiple identifiers\n", + "When you need to find different names (e.g. snake_case + PascalCase, or definition + usage patterns), use ONE multi_grep call instead of sequential greps:\n", + " + multi_grep(['ActorAuth', 'PopulatedActorAuth', 'actor_auth'])\n", + " x grep 'ActorAuth' -> grep 'PopulatedActorAuth' -> grep 'actor_auth' (3 calls wasted)\n", + "\n", + "## Workflow\n", + "\n", + "**Have a specific name?** -> grep the bare identifier.\n", + "**Need multiple name variants?** -> multi_grep with all variants in one call.\n", + "**Exploring a topic / finding files?** -> find_files.\n", + "**Got results?** -> Read the top file. Don't grep again.\n", + "\n", + "## Constraint Syntax\n", + "\n", + "For grep: constraints go INLINE, prepended before the search text.\n", + "For multi_grep: constraints go in the separate 'constraints' parameter.\n", + "\n", + "Constraints MUST match one of these formats:\n", + " Extension: '*.rs', '*.{ts,tsx}'\n", + " Directory: 'src/', 'quotes/'\n", + " Filename: 'schema.rs', 'src/main.rs'\n", + " Exclude: '!test/', '!*.spec.ts'\n", + "\n", + "! Bare words without extensions are NOT constraints. 'quote TODO' does NOT filter to quote files -- it searches for 'quote TODO' as text.\n", + " + 'schema.rs TODO' -> searches for 'TODO' in files schema.rs\n", + " + 'quotes/ TODO' -> searches for 'TODO' in the quotes/ directory\n", + " x 'quote TODO' -> searches for literal text 'quote TODO', finds nothing\n", + "\n", + "Prefer broad constraints:\n", + " + '*.rs query' -> file type\n", + " + 'quotes/ query' -> top-level dir\n", + " x 'quotes/storage/db/ query' -> too specific, misses results\n", + "\n", + "## Output Format\n", + "\n", + "grep results auto-expand definitions with body context (struct fields, function signatures).\n", + "This often provides enough information WITHOUT a follow-up Read call.\n", + "Lines marked with | are definition body context. [def] marks definition files.\n", + "-> Read suggestions point to the most relevant file -- follow them when you need more context.\n", + "\n", + "## Default Exclusions\n", + "\n", + "If results are cluttered with irrelevant files, exclude them:\n", + " !tests/ - exclude tests directory\n", + " !*.spec.ts - exclude test files\n", + " !generated/ - exclude generated code", +); + +/// FFF MCP Server -- a high performance & accuracy file finder for AI code assistants. +#[derive(Parser)] +#[command(name = "fff-mcp", version = concat!(env!("CARGO_PKG_VERSION"), " (", env!("FFF_GIT_HASH"), ")"))] +pub(crate) struct Args { + /// Base directory to index. Defaults to the current working directory. + #[arg(value_name = "PATH")] + base_path: Option, + + /// Path to the frecency database. + #[arg(long = "frecency-db")] + frecency_db_path: Option, + + /// Path to the query history database. + #[arg(long = "history-db")] + #[allow(dead_code)] + history_db_path: Option, + + /// Path-shape hint for per-session log files. + /// Each fff-mcp startup writes a fresh sibling file `++.` + #[arg(long = "log-file")] + log_file: Option, + + /// Log level (e.g. trace, debug, info, warn, error). + #[arg(long = "log-level")] + log_level: Option, + + /// Disable automatic update checks on startup. + #[arg(long = "no-update-check")] + no_update_check: bool, + + /// Disable eager mmap warmup after the initial scan. Grep results will + /// still work (files are mmap'd lazily on first access), but the first + /// search may be slightly slower. Useful on very large repos where the + /// warmup would consume too many kernel resources. + #[arg(long = "no-warmup")] + no_warmup: bool, + + /// Disable the content index built after the initial scan. + /// This makes grep calls slower but consumes less RAM (recommended to not turn off) + no_content_indexing: bool, + + /// Explicitly enable content indexing even when `--no-warmup` is set. + #[arg(long = "content-indexing")] + content_indexing: bool, + + /// Disable the background file-system watcher. Files are scanned once + /// at startup but not monitored for changes. + #[arg(long = "no-watch")] + no_watch: bool, + + /// Maximum number of files whose content is kept persistently in memory. + /// Files beyond this limit are still searchable via temporary mmaps that + /// are released after each grep. Defaults to 30 000. + /// Also settable via the FFF_MAX_CACHED_FILES environment variable. + #[arg(long = "max-cached-files", env = "FFF_MAX_CACHED_FILES")] + max_cached_files: Option, + + /// Follow symlinks during scan and watcher walks. Off by default — + /// enabling on cyclic symlink layouts can wedge the watcher. + #[arg(long = "follow-symlinks")] + follow_symlinks: bool, + + /// Run a health check and print diagnostic information, then exit. + #[arg(long = "healthcheck")] + pub(crate) healthcheck: bool, + + /// Exit after this many seconds of inactivity. 0 = never exit. + #[arg( + long = "idle-timeout-secs", + env = "FFF_MCP_IDLE_TIMEOUT_SECS", + default_value_t = 900 + )] + idle_timeout_secs: u64, +} + +/// Resolve default paths for the log file. +/// Database paths (frecency, history) must be explicitly provided via flags. +fn resolve_defaults(args: &mut Args) { + // Ensure parent directories exist for database paths when provided + for path in [&args.frecency_db_path, &args.history_db_path] + .into_iter() + .flatten() + { + if let Some(parent) = std::path::Path::new(path).parent() { + let _ = std::fs::create_dir_all(parent); + } + } + + if args.log_file.is_none() { + let home = dirs_home(); + let is_windows = cfg!(target_os = "windows"); + args.log_file = Some(if is_windows { + format!("{}\\AppData\\Local\\fff_mcp.log", home) + } else { + format!("{}/.cache/fff_mcp.log", home) + }); + } +} + +fn dirs_home() -> String { + std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .unwrap_or_else(|_| "/tmp".to_string()) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let mut args = Args::parse(); + resolve_defaults(&mut args); + + if args.healthcheck { + return healthcheck::run_healthcheck(&args); + } + + let log_file = args.log_file.as_deref().unwrap_or(""); + if let Err(e) = fff::log::init_tracing(log_file, args.log_level.as_deref(), None) { + eprintln!("Warning: Failed to init tracing: {}", e); + } + + let base_path = args.base_path.unwrap_or_else(|| { + std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .to_string() + }); + + let base_path = match Repository::discover(&base_path) { + Ok(repo) => { + if let Some(workdir) = repo.workdir() { + let git_root = workdir.to_string_lossy().to_string(); + tracing::info!("Discovered git root: {}", git_root); + git_root + } else { + tracing::info!("Git repository is bare, using base path: {}", base_path); + base_path + } + } + Err(_) => { + tracing::info!( + "No git repository found, indexing from base path: {}", + base_path + ); + base_path + } + }; + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + if let Some(frecency_db_path) = args.frecency_db_path { + match FrecencyTracker::open(&frecency_db_path) { + Ok(tracker) => { + let _ = shared_frecency.init(tracker); + } + Err(e) => { + eprintln!("Warning: Failed to init frecency db: {}", e); + } + } + } + + // Content indexing follows warmup by default (backward compat), unless + // the user explicitly opts in via --content-indexing or out via + // --no-content-indexing. + let enable_content_indexing = if args.content_indexing { + true + } else if args.no_content_indexing { + false + } else { + !args.no_warmup + }; + + // Initialize file picker (spawns background scan + watcher) + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency, + fff::FilePickerOptions { + base_path, + enable_mmap_cache: !args.no_warmup, + enable_content_indexing, + watch: !args.no_watch, + mode: FFFMode::Ai, + cache_budget: args + .max_cached_files + .map(fff::ContentCacheBudget::new_for_repo), + follow_symlinks: args.follow_symlinks, + ..Default::default() + }, + ) + .map_err(|e| format!("Failed to init file picker: {}", e))?; + + if !args.no_update_check { + update_check::spawn_update_check(); + } + + // Create and start the MCP server + let server = FffServer::new(shared_picker.clone()); + let last_activity = server.last_activity(); + let idle_timeout_secs = args.idle_timeout_secs; + + // Wait for initial scan in background — don't block server startup + let picker_clone_for_scan = shared_picker.clone(); + tokio::task::spawn_blocking(move || { + let start = std::time::Instant::now(); + loop { + let is_scanning = picker_clone_for_scan + .read() + .ok() + .and_then(|g| g.as_ref().map(|p| p.is_scan_active())) + .unwrap_or(true); + + if !is_scanning { + tracing::info!("Initial scan completed in {:?}", start.elapsed()); + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + }); + + const STARTUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + let service = match tokio::time::timeout(STARTUP_TIMEOUT, server.serve(stdio())).await { + Ok(res) => res.map_err(|e| format!("Failed to start MCP server: {}", e))?, + Err(_) => { + return Err("MCP initialize handshake did not complete within 60s".into()); + } + }; + + if idle_timeout_secs > 0 { + last_activity.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + std::sync::atomic::Ordering::Relaxed, + ); + + let last_activity_for_watchdog = last_activity.clone(); + tokio::spawn(async move { + let tick = std::time::Duration::from_secs(60); + loop { + tokio::time::sleep(tick).await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + let last = last_activity_for_watchdog.load(std::sync::atomic::Ordering::Relaxed); + if now.saturating_sub(last) >= idle_timeout_secs { + tracing::info!( + "Exiting after {}s of inactivity (idle_timeout_secs={})", + now.saturating_sub(last), + idle_timeout_secs + ); + std::process::exit(0); + } + } + }); + } + + let picker_for_shutdown = shared_picker.clone(); + tokio::spawn(async move { + tokio::signal::ctrl_c().await.ok(); + if let Ok(mut guard) = picker_for_shutdown.write() + && let Some(ref mut picker) = *guard + { + picker.stop_background_monitor(); + } + std::process::exit(0); + }); + + service.waiting().await?; + + if let Ok(mut guard) = shared_picker.write() + && let Some(ref mut picker) = *guard + { + picker.stop_background_monitor(); + } + + Ok(()) +} diff --git a/crates/fff-mcp/src/output.rs b/crates/fff-mcp/src/output.rs new file mode 100644 index 0000000..21ba7ec --- /dev/null +++ b/crates/fff-mcp/src/output.rs @@ -0,0 +1,576 @@ +//! Output formatting for MCP grep/search results. + +use fff::GrepMatch; +use fff::file_picker::FilePicker; +use fff::git::format_git_status_opt; +use fff::grep::is_import_line; +use fff::types::FileItem; + +use crate::cursor::CursorStore; + +fn frecency_word(score: i32) -> Option<&'static str> { + if score >= 100 { + Some("hot") + } else if score >= 50 { + Some("warm") + } else if score >= 10 { + Some("frequent") + } else { + None + } +} + +pub fn file_suffix(git_status: Option, frecency_score: i32) -> String { + match ( + frecency_word(frecency_score), + format_git_status_opt(git_status), + ) { + (Some(f), Some(g)) => format!(" - {f} git:{g}"), + (Some(f), None) => format!(" - {f}"), + (None, Some(g)) => format!(" git:{g}"), + (None, None) => String::new(), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutputMode { + Content, + FilesWithMatches, + Count, + Usage, +} + +impl OutputMode { + pub fn new(s: Option<&str>) -> Self { + match s { + Some("files_with_matches") => Self::FilesWithMatches, + Some("count") => Self::Count, + Some("usage") => Self::Usage, + _ => Self::Content, + } + } +} + +const LARGE_FILE_BYTES: u64 = 20_000; + +fn size_tag(bytes: u64) -> String { + if bytes < LARGE_FILE_BYTES { + String::new() + } else { + let kb = (bytes + 512) / 1024; // round + format!(" ({}KB - use offset to read relevant section)", kb) + } +} + +const MAX_PREVIEW: usize = 120; +const MAX_LINE_LEN: usize = 180; +const MAX_DEF_EXPAND_FIRST: usize = 8; +const MAX_DEF_EXPAND: usize = 5; +const MAX_FIRST_MATCH_EXPAND: usize = 8; + +fn trauncate_line_for_ai( + line: &str, + match_ranges: Option<&[(u32, u32)]>, + max_len: usize, +) -> String { + // Leading whitespace is already stripped by core (trim_whitespace option). + // Only strip trailing whitespace here. + let trimmed = line.trim_end(); + if trimmed.is_empty() { + return String::new(); + } + + if trimmed.len() <= max_len { + return trimmed.to_string(); + } + + // Use first match range to center the window + if let Some(ranges) = match_ranges + && let Some(&(match_start, match_end)) = ranges.first() + { + let match_start = match_start as usize; + let match_end = match_end as usize; + let match_len = match_end.saturating_sub(match_start); + + let budget = max_len.saturating_sub(match_len); + let before = budget / 3; + let after = budget - before; + + let win_start = match_start.saturating_sub(before); + let win_end = (match_end + after).min(trimmed.len()); + + // Clamp to char boundaries + let win_start = floor_char_boundary(trimmed, win_start); + let win_end = ceil_char_boundary(trimmed, win_end); + + let mut result = trimmed[win_start..win_end].to_string(); + if win_start > 0 { + result.insert(0, '…'); + } + if win_end < trimmed.len() { + result.push('…'); + } + return result; + } + + // No match ranges — truncate from start + let end = ceil_char_boundary(trimmed, max_len); + format!("{}…", &trimmed[..end]) +} + +fn floor_char_boundary(s: &str, index: usize) -> usize { + if index >= s.len() { + return s.len(); + } + let mut i = index; + while i > 0 && !s.is_char_boundary(i) { + i -= 1; + } + i +} + +fn ceil_char_boundary(s: &str, index: usize) -> usize { + if index >= s.len() { + return s.len(); + } + let mut i = index; + while i < s.len() && !s.is_char_boundary(i) { + i += 1; + } + i +} + +struct FileMeta<'a> { + file: &'a FileItem, + line_number: u64, + line_content: String, + is_definition: bool, + match_ranges: Vec<(u32, u32)>, + context_after: Vec, +} + +pub struct GrepFormatter<'a> { + pub matches: &'a [GrepMatch], + pub files: &'a [&'a FileItem], + pub total_matched: usize, + pub next_file_offset: usize, + pub output_mode: OutputMode, + pub max_results: usize, + pub show_context: bool, + pub auto_expand_defs: bool, + pub picker: &'a FilePicker, +} + +impl GrepFormatter<'_> { + pub fn format(&self, cursor_store: &mut CursorStore) -> String { + let GrepFormatter { + matches, + files, + total_matched, + next_file_offset, + output_mode, + max_results, + show_context, + auto_expand_defs, + picker, + } = *self; + + let items = if matches.len() > max_results { + &matches[..max_results] + } else { + matches + }; + + if output_mode == OutputMode::FilesWithMatches { + return format_files_with_matches( + items, + files, + next_file_offset, + auto_expand_defs, + cursor_store, + picker, + ); + } + + if output_mode == OutputMode::Count { + return format_count(items, files, next_file_offset, cursor_store, picker); + } + + // output_mode == usage + let mut lines: Vec = Vec::new(); + let unique_files = { + let mut seen = std::collections::HashSet::new(); + for m in items { + seen.insert(m.file_index); + } + seen.len() + }; + + let max_output_chars: usize = if output_mode == OutputMode::Usage || unique_files <= 3 { + 5000 + } else if unique_files <= 8 { + 3500 + } else { + 2500 + }; + + // File overview: collect first match per file + let file_preview = collect_file_preview(items, files, picker); + let mut content_def_file = String::new(); + let mut content_first_file = String::new(); + for fm in &file_preview { + if content_first_file.is_empty() { + content_first_file = fm.file.relative_path(picker); + } + if content_def_file.is_empty() && fm.is_definition { + content_def_file = fm.file.relative_path(picker); + } + } + + let content_suggest = if !content_def_file.is_empty() { + &content_def_file + } else { + &content_first_file + }; + if !content_suggest.is_empty() { + let file_count = file_preview.len(); + if file_count == 1 { + lines.push(format!("→ Read {} (only match)", content_suggest)); + } else if !content_def_file.is_empty() { + lines.push(format!("→ Read {} [def]", content_suggest)); + } else if file_count <= 3 { + lines.push(format!("→ Read {} (best match)", content_suggest)); + } + } + + if total_matched > items.len() { + lines.push(format!("{}/{} matches shown", items.len(), total_matched)); + } + + // Track which files already had a definition expanded + let mut def_expanded_files = std::collections::HashSet::new(); + + // Detailed content (subject to budget) + let mut char_count = 0usize; + let mut shown_count = 0usize; + let mut current_file = String::new(); + + // Reorder: definitions first, then usages, then imports (when auto-expanding) + let sorted_items: Vec = if auto_expand_defs { + let mut indices: Vec = (0..items.len()).collect(); + indices.sort_unstable_by_key(|&i| { + if items[i].is_definition { + 0 + } else if is_import_line(&items[i].line_content) { + 2 + } else { + 1 + } + }); + + indices + } else { + (0..items.len()).collect() + }; + + for &idx in &sorted_items { + let m = &items[idx]; + let file = files[m.file_index]; + let mut match_lines: Vec = Vec::new(); + + let file_rel_path = file.relative_path(picker); + if file_rel_path != current_file { + current_file = file_rel_path; + match_lines.push(current_file.to_string()); + } + + // Skip import-only lines when we already have definitions + if auto_expand_defs && is_import_line(&m.line_content) && !def_expanded_files.is_empty() + { + continue; + } + + // Context before (only when explicitly requested) + if show_context && !m.context_before.is_empty() { + let start_line = m.line_number.saturating_sub(m.context_before.len() as u64); + for (i, ctx) in m.context_before.iter().enumerate() { + match_lines.push(format!( + " {}-{}", + start_line + i as u64, + trauncate_line_for_ai(ctx, None, MAX_LINE_LEN) + )); + } + } + + // Match line + match_lines.push(format!( + " {}: {}", + m.line_number, + trauncate_line_for_ai( + &m.line_content, + Some(m.match_byte_offsets.as_ref()), + MAX_LINE_LEN + ) + )); + + // Context after (only when explicitly requested via context parameter) + if show_context && !m.context_after.is_empty() { + let start_line = m.line_number + 1; + for (i, ctx) in m.context_after.iter().enumerate() { + match_lines.push(format!( + " {}-{}", + start_line + i as u64, + trauncate_line_for_ai(ctx, None, MAX_LINE_LEN) + )); + } + match_lines.push("--".to_string()); + } + + // Auto-expand definitions with body context + let file_rel_for_expand = file.relative_path(picker); + if auto_expand_defs + && !show_context + && m.is_definition + && !m.context_after.is_empty() + && !def_expanded_files.contains(&file_rel_for_expand) + { + let expand_limit = if def_expanded_files.is_empty() { + MAX_DEF_EXPAND_FIRST + } else { + MAX_DEF_EXPAND + }; + def_expanded_files.insert(file_rel_for_expand); + let start_line = m.line_number + 1; + for (i, ctx) in m.context_after.iter().take(expand_limit).enumerate() { + if ctx.trim().is_empty() { + break; + } + match_lines.push(format!( + " {}| {}", + start_line + i as u64, + trauncate_line_for_ai(ctx, None, MAX_LINE_LEN) + )); + } + } + + let chunk = match_lines.join("\n"); + if char_count + chunk.len() > max_output_chars && shown_count > 0 { + break; + } + + char_count += chunk.len(); + lines.push(chunk); + shown_count += 1; + } + + if next_file_offset > 0 { + let cursor_id = cursor_store.store(next_file_offset); + lines.push(format!("\ncursor: {}", cursor_id)); + } + + lines.join("\n") + } +} + +fn format_files_with_matches( + items: &[GrepMatch], + files: &[&FileItem], + next_file_offset: usize, + auto_expand_defs: bool, + cursor_store: &mut CursorStore, + picker: &FilePicker, +) -> String { + let file_map = collect_file_preview(items, files, picker); + + let mut lines: Vec = Vec::new(); + let file_count = file_map.len(); + + // Find best Read target + let mut first_def_file = String::new(); + let mut first_file = String::new(); + for fm in &file_map { + if first_file.is_empty() { + first_file = fm.file.relative_path(picker); + } + if first_def_file.is_empty() && fm.is_definition { + first_def_file = fm.file.relative_path(picker); + } + } + let suggest_path = if !first_def_file.is_empty() { + &first_def_file + } else { + &first_file + }; + + if !suggest_path.is_empty() { + if file_count == 1 { + lines.push(format!( + "→ Read {} (only match — no need to search further)", + suggest_path + )); + } else if !first_def_file.is_empty() && file_count <= 5 { + lines.push(format!("→ Read {} (definition found)", suggest_path)); + } else if !first_def_file.is_empty() { + lines.push(format!("→ Read {} (definition)", suggest_path)); + } else if file_count <= 3 { + lines.push(format!("→ Read {} (best match)", suggest_path)); + } else { + lines.push(format!("→ Read {}", suggest_path)); + } + } + + let is_small_set = file_count <= 5; + let mut def_expanded_count = 0usize; + + for (file_idx, fm) in file_map.iter().enumerate() { + let is_def = fm.is_definition; + let def_tag = if is_def { " [def]" } else { "" }; + lines.push(format!( + "{}{}{}", + fm.file.relative_path(picker), + def_tag, + size_tag(fm.file.size) + )); + + // Show preview + if !fm.line_content.is_empty() && (is_def || file_idx == 0 || is_small_set) { + let ranges_ref: Option<&[(u32, u32)]> = if fm.match_ranges.is_empty() { + None + } else { + Some(&fm.match_ranges) + }; + lines.push(format!( + " {}: {}", + fm.line_number, + trauncate_line_for_ai(&fm.line_content, ranges_ref, MAX_PREVIEW) + )); + + // Auto-expand body context + if auto_expand_defs && !fm.context_after.is_empty() { + let expand_limit = if is_def { + let limit = if def_expanded_count == 0 { + MAX_DEF_EXPAND_FIRST + } else { + MAX_DEF_EXPAND + }; + def_expanded_count += 1; + limit + } else if is_small_set && file_idx == 0 { + MAX_FIRST_MATCH_EXPAND + } else if is_small_set { + MAX_DEF_EXPAND + } else { + 0 + }; + + if expand_limit > 0 { + let start_line = fm.line_number + 1; + for (i, ctx) in fm.context_after.iter().take(expand_limit).enumerate() { + if ctx.trim().is_empty() { + break; + } + lines.push(format!( + " {}| {}", + start_line + i as u64, + trauncate_line_for_ai(ctx, None, MAX_PREVIEW) + )); + } + } + } + } + } + + if next_file_offset > 0 { + let cursor_id = cursor_store.store(next_file_offset); + lines.push(format!("\ncursor: {}", cursor_id)); + } + + lines.join("\n") +} + +fn format_count( + items: &[GrepMatch], + files: &[&FileItem], + next_file_offset: usize, + cursor_store: &mut CursorStore, + picker: &FilePicker, +) -> String { + let mut counts: std::collections::HashMap = std::collections::HashMap::new(); + let mut order: Vec = Vec::new(); + for m in items { + let file = files[m.file_index]; + let path = file.relative_path(picker); + let count = counts.entry(path.to_string()).or_insert_with(|| { + order.push(path.to_string()); + 0 + }); + *count += 1; + } + + let mut lines: Vec = Vec::new(); + for path in &order { + lines.push(format!("{}: {}", path, counts[path.as_str()])); + } + if next_file_offset > 0 { + let cursor_id = cursor_store.store(next_file_offset); + lines.push(format!("\ncursor: {}", cursor_id)); + } + lines.join("\n") +} + +fn collect_file_preview<'a>( + items: &[GrepMatch], + files: &[&'a FileItem], + picker: &FilePicker, +) -> Vec> { + let mut file_preview: Vec> = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for m in items { + let file = files[m.file_index]; + if seen.insert(file.relative_path(picker)) { + file_preview.push(FileMeta { + file, + line_number: m.line_number, + line_content: m.line_content.clone(), + is_definition: m.is_definition, + match_ranges: m.match_byte_offsets.iter().copied().collect(), + context_after: m.context_after.clone(), + }); + } + } + file_preview +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trunc_strips_trailing_whitespace() { + // Leading whitespace is now stripped by core's trim_whitespace option. + // This function only strips trailing whitespace. + assert_eq!(trauncate_line_for_ai("foo()", None, 180), "foo()"); + assert_eq!(trauncate_line_for_ai("bar ", None, 180), "bar"); + assert_eq!(trauncate_line_for_ai(" ", None, 180), ""); + } + + #[test] + fn trunc_preserves_pre_trimmed_match_ranges() { + // Core already stripped leading whitespace and adjusted offsets, + // so "hello" arrives with match at bytes 0..5. + let line = "hello"; + let ranges = [(0, 5)]; + let result = trauncate_line_for_ai(line, Some(&ranges), 180); + assert_eq!(result, "hello"); + } + + #[test] + fn trunc_long_line_centered() { + // Core already stripped leading whitespace; offsets are pre-adjusted. + let line = format!("match_here{}", "x".repeat(200)); + let ranges = [(0u32, 10u32)]; + let result = trauncate_line_for_ai(&line, Some(&ranges), 50); + assert!(result.contains("match_here")); + assert!(result.len() <= 55); // budget + ellipsis chars + } +} diff --git a/crates/fff-mcp/src/server.rs b/crates/fff-mcp/src/server.rs new file mode 100644 index 0000000..c4affa3 --- /dev/null +++ b/crates/fff-mcp/src/server.rs @@ -0,0 +1,768 @@ +use crate::cursor::CursorStore; +use crate::output::{GrepFormatter, OutputMode, file_suffix}; +use fff::grep::{GrepMode, GrepSearchOptions, has_regex_metacharacters}; +use fff::types::{FileItem, PaginationArgs}; +use fff::{FuzzySearchOptions, QueryParser, SharedFilePicker}; +use fff_query_parser::AiGrepConfig; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::*; +use rmcp::{ServerHandler, schemars, tool, tool_handler, tool_router}; +use std::borrow::Cow; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const SCAN_READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +fn normalize_max_results(raw: Option, default: usize) -> usize { + match raw { + None => default, + Some(v) if v <= 0.0 || !v.is_finite() => default, + Some(v) => (v.round() as usize).max(1), + } +} + +fn cleanup_fuzzy_query(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if !matches!(c, ':' | '-' | '_') { + out.extend(c.to_lowercase()); + } + } + out +} + +fn make_grep_options( + output_mode: OutputMode, + mode: GrepMode, + file_offset: usize, + context: Option, +) -> (GrepSearchOptions, bool) { + let is_usage = output_mode == OutputMode::Usage; + let matches_per_file = match output_mode { + OutputMode::FilesWithMatches => 1, + _ if is_usage => 8, + _ => 10, + }; + let ctx_lines = if is_usage { + context.unwrap_or(1) + } else { + context.unwrap_or(0) + }; + let auto_expand = !is_usage && ctx_lines == 0; + let after_ctx = if auto_expand { 8 } else { ctx_lines }; + + ( + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: matches_per_file, + smart_case: true, + file_offset, + page_limit: 50, + mode, + time_budget_ms: 0, + before_context: ctx_lines, + after_context: after_ctx, + classify_definitions: true, + trim_whitespace: true, + abort_signal: None, + }, + auto_expand, + ) +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct FindFilesParams { + /// Fuzzy search query. Supports path prefixes and glob constraints. + // `pattern` alias for consistency with grep's alias and the common + // file-search parameter name (#311). + #[serde(alias = "pattern")] + pub query: String, + /// Max results (default 20). + #[serde(rename = "maxResults")] + // this has to be float because llms are stupid + pub max_results: Option, + /// Cursor from previous result. Only use if previous results weren't sufficient. + pub cursor: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct GrepParams { + /// Search text or regex query with optional constraint prefixes. + /// Matches within single lines only — use ONE specific term, not multiple words. + // `pattern` alias: LLMs that have seen multi_grep (which uses `patterns`) + // routinely call grep with `pattern`; accept it instead of erroring out + // with an unhelpful "missing field `query`" (#311). + #[serde(alias = "pattern")] + pub query: String, + /// Max matching lines (default 20). + #[serde(rename = "maxResults")] + pub max_results: Option, // this has to be float because llms are stupid + /// Cursor from previous result. Only use if previous results weren't sufficient. + pub cursor: Option, + /// Output format (default 'content'). + pub output_mode: Option, +} + +fn deserialize_patterns<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::de; + + struct PatternsVisitor; + + impl<'de> de::Visitor<'de> for PatternsVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a string, an array of strings, or a stringified JSON array") + } + + fn visit_str(self, v: &str) -> Result { + // Try to parse as JSON array first + if v.starts_with('[') + && let Ok(parsed) = serde_json::from_str::>(v) + { + return Ok(parsed); + } + Ok(vec![v.to_string()]) + } + + fn visit_string(self, v: String) -> Result { + if v.starts_with('[') + && let Ok(parsed) = serde_json::from_str::>(&v) + { + return Ok(parsed); + } + Ok(vec![v]) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut values = Vec::new(); + while let Some(value) = seq.next_element::()? { + values.push(value); + } + Ok(values) + } + } + + deserializer.deserialize_any(PatternsVisitor) +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct MultiGrepParams { + /// Patterns to match (OR logic). Include all naming conventions: snake_case, PascalCase, camelCase. + #[serde(deserialize_with = "deserialize_patterns")] + pub patterns: Vec, + /// File constraints (e.g. '*.{ts,tsx} !test/'). ALWAYS provide when possible. + pub constraints: Option, + /// Max matching lines (default 20). + #[serde(rename = "maxResults")] + pub max_results: Option, + /// Cursor from previous result. + pub cursor: Option, + /// Output format (default 'content'). + pub output_mode: Option, + /// Context lines before/after each match. + pub context: Option, +} + +#[derive(Clone)] +pub struct FffServer { + picker: SharedFilePicker, + cursor_store: Arc>, + update_notice_sent: Arc, + last_activity: Arc, + scan_ready: Arc, +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl FffServer { + pub fn new(picker: SharedFilePicker) -> Self { + Self { + picker, + cursor_store: Arc::new(Mutex::new(CursorStore::new())), + update_notice_sent: Arc::new(AtomicBool::new(false)), + last_activity: Arc::new(AtomicU64::new(now_secs())), + scan_ready: Arc::new(AtomicBool::new(false)), + } + } + + pub fn last_activity(&self) -> Arc { + self.last_activity.clone() + } + + fn bump_activity(&self) { + self.last_activity.store(now_secs(), Ordering::Relaxed); + } + + fn wait_for_scan(&self, timeout: std::time::Duration) -> Result<(), ErrorData> { + if self.scan_ready.load(Ordering::Relaxed) { + return Ok(()); + } + + let deadline = std::time::Instant::now() + timeout; + + loop { + let is_scanning = self + .picker + .read() + .ok() + .as_ref() + .and_then(|g| g.as_ref()) + .map(|p| p.is_scan_active()) + .unwrap_or(true); + + if !is_scanning { + self.scan_ready.store(true, Ordering::Relaxed); + return Ok(()); + } + if std::time::Instant::now() >= deadline { + return Err(ErrorData::internal_error( + "Index is still building; retry shortly", + None, + )); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + + fn lock_cursors(&self) -> Result, ErrorData> { + self.cursor_store.lock().map_err(|e| { + ErrorData::internal_error(format!("Failed to acquire cursor store lock: {e}"), None) + }) + } + + fn maybe_append_update_notice(&self, result: &mut CallToolResult) { + if self.update_notice_sent.swap(true, Ordering::Relaxed) { + return; + } + let notice = crate::update_check::get_update_notice(); + if notice.is_empty() { + // Reset so the next call can try again (check may still be in flight) + self.update_notice_sent.store(false, Ordering::Relaxed); + return; + } + result.content.push(Content::text(notice)); + } + + fn perform_grep( + &self, + query: &str, + mode: GrepMode, + max_results: usize, + cursor_id: Option<&str>, + output_mode: OutputMode, + context: Option, + ) -> Result { + let file_offset = cursor_id + .and_then(|id| self.cursor_store.lock().ok()?.get(id)) + .unwrap_or(0); + + let (options, auto_expand) = make_grep_options(output_mode, mode, file_offset, context); + let ctx_lines = options.before_context; + + // Acquire picker lock once for the entire operation. + let guard = self.picker.read().map_err(|e| { + ErrorData::internal_error(format!("Failed to acquire picker lock: {e}"), None) + })?; + let picker = guard + .as_ref() + .ok_or_else(|| ErrorData::internal_error("File picker not initialized", None))?; + + let parser = QueryParser::new(AiGrepConfig); + let parsed = parser.parse(query); + let result = picker.grep(&parsed, &options); + + if result.matches.is_empty() && file_offset == 0 { + // Auto-retry: try broadening multi-word queries by dropping first non-constraint word + let parts: Vec<&str> = query.split_whitespace().collect(); + if parts.len() >= 2 { + let first_word = parts[0]; + let is_valid_constraint = first_word.starts_with('!') + || first_word.starts_with('*') + || first_word.ends_with('/'); + + if !is_valid_constraint { + let rest_query = parts[1..].join(" "); + let rest_parsed = parser.parse(&rest_query); + + let rest_text = rest_parsed.grep_text(); + let retry_mode = if has_regex_metacharacters(&rest_text) { + GrepMode::Regex + } else { + mode + }; + + let (retry_options, _) = make_grep_options(output_mode, retry_mode, 0, context); + let retry_result = picker.grep(&rest_parsed, &retry_options); + + if !retry_result.matches.is_empty() && retry_result.matches.len() <= 10 { + let mut cs = self.lock_cursors()?; + let text = &GrepFormatter { + matches: &retry_result.matches, + files: &retry_result.files, + total_matched: retry_result.matches.len(), + next_file_offset: retry_result.next_file_offset, + output_mode, + max_results, + show_context: ctx_lines > 0, + auto_expand_defs: auto_expand, + picker, + } + .format(&mut cs); + return Ok(CallToolResult::success(vec![Content::text(format!( + "0 matches for '{}'. Auto-broadened to '{}':\n{}", + query, rest_query, text + ))])); + } + } + } + + // Fuzzy fallback for typo tolerance + let fuzzy_query = cleanup_fuzzy_query(query); + let (fuzzy_options, _) = make_grep_options(output_mode, GrepMode::Fuzzy, 0, Some(0)); + let fuzzy_parsed = parser.parse(&fuzzy_query); + let fuzzy_result = picker.grep(&fuzzy_parsed, &fuzzy_options); + + if !fuzzy_result.matches.is_empty() { + let mut lines: Vec = Vec::new(); + lines.push(format!( + "0 exact matches. {} approximate:", + fuzzy_result.matches.len() + )); + let mut current_file = String::new(); + for m in fuzzy_result.matches.iter().take(3) { + let file = fuzzy_result.files[m.file_index]; + let file_rel = file.relative_path(picker); + if file_rel != current_file { + current_file = file_rel; + lines.push(current_file.to_string()); + } + lines.push(format!(" {}: {}", m.line_number, m.line_content)); + } + return Ok(CallToolResult::success(vec![Content::text( + lines.join("\n"), + )])); + } + + // File path fallback: if query looks like a path, suggest the matching file + if query.contains('/') { + let file_parser = QueryParser::default(); + let file_query = file_parser.parse(query); + let file_opts = FuzzySearchOptions { + max_threads: 0, + current_file: None, + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 1, + }, + }; + let file_result = picker.fuzzy_search(&file_query, None, file_opts); + if let (Some(top), Some(score)) = + (file_result.items.first(), file_result.scores.first()) + { + // Only suggest when the match is strong enough. + let query_len = query.len() as i32; + if score.base_score > query_len * 10 { + return Ok(CallToolResult::success(vec![Content::text(format!( + "0 content matches. But there is a relevant file path: {}", + top.relative_path(picker) + ))])); + } + } + } + + return Ok(CallToolResult::success(vec![Content::text( + "0 matches.".to_string(), + )])); + } + + if result.matches.is_empty() { + return Ok(CallToolResult::success(vec![Content::text( + "0 matches.".to_string(), + )])); + } + + let mut cs = self.lock_cursors()?; + let text = &GrepFormatter { + matches: &result.matches, + files: &result.files, + total_matched: result.matches.len(), + next_file_offset: result.next_file_offset, + output_mode, + max_results, + show_context: ctx_lines > 0, + auto_expand_defs: auto_expand, + picker, + } + .format(&mut cs); + + Ok(CallToolResult::success(vec![Content::text(text)])) + } +} + +#[tool_router] +impl FffServer { + /// Fuzzy file search by name. Searches FILE NAMES, not file contents. + /// Use it when you need to find a file, not a definition. + /// Use grep instead for searching code content (definitions, usage patterns). + /// Supports fuzzy matching, path prefixes ('shc/'), and glob constraints. + /// IMPORTANT: Keep queries SHORT — prefer 1-2 terms max. + #[tool( + name = "find_files", + description = "Fuzzy file search by name. Searches FILE NAMES, not file contents. Use it when you need to find a file, not a definition. Use grep instead for searching code content (definitions, usage patterns). Supports fuzzy matching, path prefixes ('src/'), and glob constraints ('name **/src/*.{ts,tsx} !test/'). IMPORTANT: Keep queries SHORT — prefer 1-2 terms max. Multiple words are a waterfall (each narrows results), NOT OR. If unsure, start broad with 1 term and refine." + )] + fn find_files( + &self, + Parameters(params): Parameters, + ) -> Result { + self.bump_activity(); + self.wait_for_scan(SCAN_READY_TIMEOUT)?; + + let max_results = normalize_max_results(params.max_results, 20); + let query = ¶ms.query; + + let page_offset = params + .cursor + .as_deref() + .and_then(|id| self.cursor_store.lock().ok()?.get(id)) + .unwrap_or(0); + + let guard = self.picker.read().map_err(|e| { + ErrorData::internal_error(format!("Failed to acquire picker lock: {e}"), None) + })?; + let picker = guard + .as_ref() + .ok_or_else(|| ErrorData::internal_error("File picker not initialized", None))?; + let base_path = picker.base_path(); + let make_opts = |offset: usize| FuzzySearchOptions { + max_threads: 0, + current_file: None, + project_path: Some(base_path), + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset, + limit: max_results, + }, + }; + + let parser = QueryParser::default(); + let fff_query = parser.parse(query); + let result = picker.fuzzy_search(&fff_query, None, make_opts(page_offset)); + let total_files = result.total_files; + + // Auto-retry with fewer terms if 3+ words return 0 results + let words: Vec<&str> = query.split_whitespace().collect(); + let shorter = words.get(..2).map(|w| w.join(" ")); + + let (items, scores, total_matched) = + if result.items.is_empty() && words.len() >= 3 && page_offset == 0 { + if let Some(shorter) = &shorter { + let shorter_query = parser.parse(shorter); + let retry = picker.fuzzy_search(&shorter_query, None, make_opts(0)); + + (retry.items, retry.scores, retry.total_matched) + } else { + (result.items, result.scores, result.total_matched) + } + } else { + (result.items, result.scores, result.total_matched) + }; + + if items.is_empty() { + return Ok(CallToolResult::success(vec![Content::text(format!( + "0 results ({} indexed)", + total_files + ))])); + } + + let mut lines: Vec = Vec::new(); + let top_item = items[0]; + let is_exact_match = scores[0].exact_match; + + if page_offset == 0 { + if is_exact_match { + lines.push(format!( + "→ Read {} (exact match!)", + top_item.relative_path(picker) + )); + } else if scores.len() < 2 || scores[0].total > scores[1].total.saturating_mul(2) { + lines.push(format!( + "→ Read {} (best match — Read this file directly)", + top_item.relative_path(picker) + )); + } + } + + let next_offset = page_offset + items.len(); + let has_more = next_offset < total_matched; + + if has_more { + lines.push(format!("{}/{} matches", items.len(), total_matched)); + } + + for item in &items { + lines.push(format!( + "{}{}", + item.relative_path(picker), + file_suffix(item.git_status, item.total_frecency_score()) + )); + } + + if has_more { + let mut cs = self.lock_cursors()?; + let cursor_id = cs.store(next_offset); + lines.push(format!("cursor: {}", cursor_id)); + } + + let mut result = CallToolResult::success(vec![Content::text(lines.join("\n"))]); + self.maybe_append_update_notice(&mut result); + Ok(result) + } + + /// Search file contents for text patterns. This is the DEFAULT search tool. + /// Prefer plain text over regex. Filter files with constraints. + #[tool( + name = "grep", + description = "Search file contents. Search for bare identifiers (e.g. 'InProgressQuote', 'ActorAuth'), NOT code syntax or regex. Filter files with constraints (e.g. '*.rs query', 'src/ query'). Use filename, directory (ending with /) or glob expressions to prefilter. See server instructions for constraint syntax and core rules." + )] + fn grep( + &self, + Parameters(params): Parameters, + ) -> Result { + self.bump_activity(); + self.wait_for_scan(SCAN_READY_TIMEOUT)?; + + let max_results = normalize_max_results(params.max_results, 20); + let output_mode = OutputMode::new(params.output_mode.as_deref()); + + let parsed = QueryParser::new(AiGrepConfig).parse(¶ms.query); + let grep_text = parsed.grep_text(); + + let mode = if has_regex_metacharacters(&grep_text) { + GrepMode::Regex + } else { + GrepMode::PlainText + }; + + let mut result = self.perform_grep( + ¶ms.query, + mode, + max_results, + params.cursor.as_deref(), + output_mode, + None, + )?; + self.maybe_append_update_notice(&mut result); + Ok(result) + } + + /// Search file contents for lines matching ANY of multiple patterns (OR logic). + /// Patterns are literal text — NEVER escape special characters. + #[tool( + name = "multi_grep", + description = "Search file contents for lines matching ANY of multiple patterns (OR logic). IMPORTANT: This returns files where ANY query matches, NOT all patterns. Patterns are literal text — NEVER escape special characters (no \\( \\) \\. etc). Faster than regex alternation for literal text. See server instructions for constraint syntax." + )] + fn multi_grep( + &self, + Parameters(params): Parameters, + ) -> Result { + self.bump_activity(); + self.wait_for_scan(SCAN_READY_TIMEOUT)?; + + let mut result = self.multi_grep_inner(params)?; + self.maybe_append_update_notice(&mut result); + Ok(result) + } +} + +impl FffServer { + fn multi_grep_inner(&self, params: MultiGrepParams) -> Result { + let max_results = normalize_max_results(params.max_results, 20); + let context = params.context.map(|v| v.round() as usize); + let output_mode = OutputMode::new(params.output_mode.as_deref()); + + let file_offset = params + .cursor + .as_deref() + .and_then(|id| self.cursor_store.lock().ok()?.get(id)) + .unwrap_or(0); + + let (options, auto_expand) = + make_grep_options(output_mode, GrepMode::PlainText, file_offset, context); + + let ctx_lines = options.before_context; + let constraint_query = params.constraints.as_deref().unwrap_or(""); + let guard = self.picker.read().map_err(|e| { + ErrorData::internal_error(format!("Failed to acquire picker lock: {e}"), None) + })?; + let picker = guard + .as_ref() + .ok_or_else(|| ErrorData::internal_error("File picker not initialized", None))?; + let patterns_refs: Vec<&str> = params.patterns.iter().map(|s| s.as_str()).collect(); + + let parser = fff_query_parser::QueryParser::new(fff_query_parser::AiGrepConfig); + let parsed_constraints = parser.parse(constraint_query); + let constraints = parsed_constraints.constraints.as_slice(); + + let result = picker.multi_grep(&patterns_refs, constraints, &options); + let file_refs: Vec<&FileItem> = result.files.to_vec(); + + if result.matches.is_empty() && file_offset == 0 { + // Fallback: try individual patterns with plain grep + let (fallback_options, _) = + make_grep_options(output_mode, GrepMode::PlainText, 0, context); + + let fallback_options = GrepSearchOptions { + time_budget_ms: 3000, + before_context: 0, + ..fallback_options + }; + + for pat in ¶ms.patterns { + let full_query: Cow = if !constraint_query.is_empty() { + Cow::Owned(format!("{} {}", constraint_query, pat)) + } else { + Cow::Borrowed(pat) + }; + + let parsed = parser.parse(&full_query); + let fb_result = picker.grep(&parsed, &fallback_options); + + if !fb_result.matches.is_empty() { + let fb_file_refs: Vec<&FileItem> = fb_result.files.to_vec(); + let mut cs = self.lock_cursors()?; + let text = &GrepFormatter { + matches: &fb_result.matches, + files: &fb_file_refs, + total_matched: fb_result.matches.len(), + next_file_offset: fb_result.next_file_offset, + output_mode, + max_results, + show_context: false, + auto_expand_defs: auto_expand, + picker, + } + .format(&mut cs); + return Ok(CallToolResult::success(vec![Content::text(format!( + "0 multi-pattern matches. Plain grep fallback for \"{}\":\n{}", + pat, text + ))])); + } + } + + return Ok(CallToolResult::success(vec![Content::text( + "0 matches.".to_string(), + )])); + } + + if result.matches.is_empty() { + return Ok(CallToolResult::success(vec![Content::text( + "0 matches.".to_string(), + )])); + } + + let mut cs = self.lock_cursors()?; + let text = &GrepFormatter { + matches: &result.matches, + files: &file_refs, + total_matched: result.matches.len(), + next_file_offset: result.next_file_offset, + output_mode, + max_results, + show_context: ctx_lines > 0, + auto_expand_defs: auto_expand, + picker, + } + .format(&mut cs); + + Ok(CallToolResult::success(vec![Content::text(text)])) + } +} + +#[tool_handler] +impl ServerHandler for FffServer { + fn get_info(&self) -> ServerInfo { + let notice = crate::update_check::get_update_notice(); + let instructions = if notice.is_empty() { + crate::MCP_INSTRUCTIONS.to_string() + } else { + format!("{}{}", crate::MCP_INSTRUCTIONS, notice) + }; + + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info(Implementation::new("fff", env!("CARGO_PKG_VERSION"))) + .with_instructions(instructions) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_max_results_none_uses_default() { + assert_eq!(normalize_max_results(None, 20), 20); + } + + #[test] + fn normalize_max_results_zero_uses_default() { + // Issue #400: `maxResults: 0` must not return zero items for grep + // while `find_files` returns the full set. Both tools now map 0 to + // the default limit. + assert_eq!(normalize_max_results(Some(0.0), 20), 20); + } + + #[test] + fn normalize_max_results_negative_uses_default() { + assert_eq!(normalize_max_results(Some(-5.0), 20), 20); + } + + #[test] + fn normalize_max_results_non_finite_uses_default() { + assert_eq!(normalize_max_results(Some(f64::NAN), 20), 20); + assert_eq!(normalize_max_results(Some(f64::INFINITY), 20), 20); + } + + #[test] + fn normalize_max_results_rounds_and_clamps() { + assert_eq!(normalize_max_results(Some(0.4), 20), 1); + assert_eq!(normalize_max_results(Some(10.0), 20), 10); + assert_eq!(normalize_max_results(Some(10.7), 20), 11); + } + + #[test] + fn grep_params_accepts_pattern_alias() { + // Issue #311: LLMs flip between `query` and `pattern`; accept both. + let via_query: GrepParams = + serde_json::from_str(r#"{"query":"foo"}"#).expect("query field"); + assert_eq!(via_query.query, "foo"); + + let via_pattern: GrepParams = + serde_json::from_str(r#"{"pattern":"foo"}"#).expect("pattern alias"); + assert_eq!(via_pattern.query, "foo"); + } + + #[test] + fn find_files_params_accepts_pattern_alias() { + let via_query: FindFilesParams = + serde_json::from_str(r#"{"query":"foo"}"#).expect("query field"); + assert_eq!(via_query.query, "foo"); + + let via_pattern: FindFilesParams = + serde_json::from_str(r#"{"pattern":"foo"}"#).expect("pattern alias"); + assert_eq!(via_pattern.query, "foo"); + } +} diff --git a/crates/fff-mcp/src/update_check.rs b/crates/fff-mcp/src/update_check.rs new file mode 100644 index 0000000..0cfd909 --- /dev/null +++ b/crates/fff-mcp/src/update_check.rs @@ -0,0 +1,78 @@ +//! Background update checker — compares the embedded build hash against +//! the latest GitHub release tag to surface upgrade notices in MCP instructions. + +use std::sync::OnceLock; + +const REPO: &str = "dmtrKovalenko/fff.nvim"; +const BUILD_HASH: &str = env!("FFF_GIT_HASH"); + +/// Holds the result of the update check (empty string = up to date or check failed). +static UPDATE_NOTICE: OnceLock = OnceLock::new(); + +/// Returns the update notice if the check has completed, empty string otherwise. +pub fn get_update_notice() -> &'static str { + UPDATE_NOTICE.get().map(|s| s.as_str()).unwrap_or("") +} + +/// Kick off the update check in a background thread so it never blocks the server. +pub fn spawn_update_check() { + std::thread::spawn(|| { + let notice = check_latest_release(); + let _ = UPDATE_NOTICE.set(notice); + }); +} + +/// Fetch the latest release tag from GitHub and compare against the build hash. +fn check_latest_release() -> String { + match fetch_latest_tag() { + Ok(tag) => compare_versions(BUILD_HASH, &tag), + Err(_) => String::new(), + } +} + +/// Compare a build hash against a release tag. +/// Returns an update notice string, or empty if up-to-date. +fn compare_versions(build_hash: &str, release_tag: &str) -> String { + let tag = release_tag.trim(); + if tag.is_empty() || build_hash == "unknown" { + return String::new(); + } + + let our_short = &build_hash[..build_hash.len().min(tag.len())]; + if our_short == tag { + return String::new(); + } + + format!( + "\n[fff update available: `curl -fsSL https://raw.githubusercontent.com/{REPO}/main/install-mcp.sh | bash`]\n" + ) +} + +/// Shell out to curl to fetch the latest release tag name from GitHub API. +fn fetch_latest_tag() -> Result> { + let output = std::process::Command::new("curl") + .args([ + "-fsSL", + "--max-time", + "5", + "-H", + "Accept: application/vnd.github.v3+json", + &format!("https://api.github.com/repos/{REPO}/releases?per_page=1"), + ]) + .output()?; + + if !output.status.success() { + return Err("curl failed".into()); + } + + let body = String::from_utf8(output.stdout)?; + let releases: Vec = serde_json::from_str(&body)?; + let tag = releases + .first() + .and_then(|r| r.get("tag_name")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + Ok(tag) +} diff --git a/crates/fff-nvim/Cargo.toml b/crates/fff-nvim/Cargo.toml new file mode 100644 index 0000000..eb24826 --- /dev/null +++ b/crates/fff-nvim/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "fff-nvim" +version = "0.9.6" +edition = "2024" + +[lib] +path = "src/lib.rs" +crate-type = ["cdylib", "rlib"] + +[features] +# Pure-Rust walker by default; zlob is opt-in (needs Zig). +default = ["ripgrep"] +ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep", "dep:ignore"] +zlob = ["fff/zlob", "fff-query-parser/zlob", "dep:zlob"] + +[dependencies] +# Workspace dependencies +ahash = { workspace = true } +tracing = { workspace = true } + +# Local crates +fff = { package = "fff-search", path = "../fff-core", version = "0.9.6", default-features = false, features = [ + "mimalloc-collect", +] } +fff-query-parser = { path = "../fff-query-parser", version = "0.9.6", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +ctrlc = "3.4.2" +git2 = { workspace = true } +ignore = { version = "0.4.22", optional = true } +mimalloc = { version = "0.1.47", features = ["local_dynamic_tls"] } +mlua = { version = "0.11.1", features = ["module", "luajit"] } +once_cell = "1.20.2" +zlob = { workspace = true, optional = true } + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +rand = { version = "0.8", features = ["small_rng"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[[bench]] +name = "fuzzy_search" +path = "benches/fuzzy_search_bench.rs" +harness = false + +[[bench]] +name = "grep_bench" +path = "benches/grep_bench.rs" +harness = false + +[[bench]] +name = "query_tracker" +path = "benches/query_tracker_bench.rs" +harness = false + +[[bench]] +name = "scan" +path = "benches/scan_bench.rs" +harness = false diff --git a/crates/fff-nvim/benches/fuzzy_search_bench.rs b/crates/fff-nvim/benches/fuzzy_search_bench.rs new file mode 100644 index 0000000..e0ccfee --- /dev/null +++ b/crates/fff-nvim/benches/fuzzy_search_bench.rs @@ -0,0 +1,661 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use fff::file_picker::{FFFMode, FilePicker}; +use fff::types::PaginationArgs; +use fff::{ + FilePickerOptions, FuzzySearchOptions, GrepMode, GrepSearchOptions, QueryParser, + SharedFilePicker, SharedFrecency, +}; +use std::path::PathBuf; +use std::time::Duration; + +fn init_tracing() { + // use tracing_subscriber::EnvFilter; + // use tracing_subscriber::fmt; + // let _ = fmt() + // .with_env_filter( + // EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + // ) + // .with_target(false) + // .with_thread_ids(true) + // .with_line_number(true) + // .try_init(); +} + +fn init_file_picker_internal( + path: &str, + shared_picker: &SharedFilePicker, + shared_frecency: &SharedFrecency, +) -> Result<(), String> { + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: path.to_string(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .map_err(|e| format!("Failed to create FilePicker: {:?}", e)) +} + +fn wait_for_scan_completion( + shared_picker: &SharedFilePicker, + timeout_secs: u64, +) -> Result { + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let mut last_log = std::time::Instant::now(); + let mut iteration = 0; + + loop { + iteration += 1; + + { + let picker_guard = shared_picker + .read() + .map_err(|_| "Failed to acquire read lock")?; + if let Some(ref picker) = *picker_guard { + let is_scanning = picker.is_scan_active(); + let file_count = picker.get_files().len(); + + // Log progress every 2 seconds + if last_log.elapsed() >= Duration::from_secs(2) { + eprintln!( + " [{:.1}s] Scanning: {}, Files: {}, Iterations: {}", + start.elapsed().as_secs_f32(), + is_scanning, + file_count, + iteration + ); + last_log = std::time::Instant::now(); + } + + if !is_scanning && file_count > 0 { + eprintln!( + " ✓ Scan complete after {:.2}s: {} files found", + start.elapsed().as_secs_f32(), + file_count + ); + return Ok(file_count); + } + } else { + if iteration % 100 == 0 { + eprintln!( + " [{:.1}s] FilePicker is None (iteration {})", + start.elapsed().as_secs_f32(), + iteration + ); + } + } + } + + if start.elapsed() > timeout { + return Err(format!( + "Scan timed out after {} seconds (iteration {})", + timeout_secs, iteration + )); + } + + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn setup_once() -> Result<(SharedFilePicker, SharedFrecency), String> { + init_tracing(); + + let big_repo_path = PathBuf::from("./big-repo"); + if !big_repo_path.exists() { + return Err("./big-repo directory does not exist. Run git clone https://github.com/torvalds/linux.git big-repo".to_string()); + } + + let canonical_path = fff::path_utils::canonicalize(&big_repo_path) + .map_err(|e| format!("Failed to canonicalize path: {}", e))?; + eprintln!(" Path: {:?}", canonical_path); + + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + init_file_picker_internal( + &canonical_path.to_string_lossy(), + &shared_picker, + &shared_frecency, + )?; + + eprintln!(" Waiting for background scan to complete..."); + let file_count = wait_for_scan_completion(&shared_picker, 120)?; + eprintln!( + " ✓ Indexed {} files (will be reused for all benchmarks)\n", + file_count + ); + + Ok((shared_picker, shared_frecency)) +} + +fn bench_search_queries(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprint!("Failed to setup picker {e:?}"); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("search"); + group.sample_size(100); + + let test_queries = vec![ + ("short", "mod"), + ("medium", "controller"), + ("long", "user_authentication"), + ("typo", "contrlr"), + ("partial", "src/lib"), + ]; + + let parser = QueryParser::default(); + + for (name, query) in test_queries { + let parsed = parser.parse(query); + group.bench_with_input(BenchmarkId::new("query", name), &query, |b, &_query| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + results.total_matched + }); + }); + } + + group.finish(); +} + +/// Benchmark search with different thread counts +fn bench_search_thread_scaling(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping thread scaling benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("thread_scaling"); + group.sample_size(100); + + let query = "controller"; + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let thread_counts = vec![1, 2, 4, 8]; + + for threads in thread_counts { + group.bench_with_input( + BenchmarkId::from_parameter(threads), + &threads, + |b, &threads| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: threads, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + results.total_matched + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark search with different result limits +fn bench_search_result_limits(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping result limit benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("result_limits"); + group.sample_size(100); + + let query = "mod"; + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let result_limits = vec![10, 50, 100, 500]; + + for limit in result_limits { + group.bench_with_input(BenchmarkId::from_parameter(limit), &limit, |b, &limit| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { offset: 0, limit }, + }, + ); + results.total_matched + }); + }); + } + + group.finish(); +} + +/// Benchmark search algorithm performance with queries of varying selectivity +fn bench_search_scalability(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping scalability benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + if picker.get_files().len() < 1000 { + eprintln!( + "Skipping scalability benchmark: need at least 1000 files, got {}", + picker.get_files().len() + ); + return; + } + + let mut group = c.benchmark_group("search_scalability"); + group.sample_size(50); + + let parser = QueryParser::default(); + let selectivity_queries = vec![ + ("broad_a", "a"), + ("medium_mod", "mod"), + ("narrow_controller", "controller"), + ("very_narrow_user_auth", "user_authentication"), + ]; + + for (name, query) in selectivity_queries { + let parsed = parser.parse(query); + group.bench_with_input(BenchmarkId::from_parameter(name), &name, |b, _| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + results.total_matched + }); + }); + } + + group.finish(); +} + +/// Benchmark search performance with different ordering modes +fn bench_search_ordering(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping ordering benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("ordering"); + group.sample_size(100); + + let parser = QueryParser::default(); + let parsed_controller = parser.parse("controller"); + let parsed_mod = parser.parse("mod"); + + // Benchmark normal order (descending) + group.bench_function("normal_order", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_controller), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + results.total_matched + }); + }); + + // Benchmark reverse order (ascending) + group.bench_function("reverse_order", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_controller), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + results.total_matched + }); + }); + + // Benchmark with large result set + group.bench_function("normal_order_large", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_mod), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 500, + }, + }, + ); + results.total_matched + }); + }); + + group.bench_function("reverse_order_large", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_mod), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 500, + }, + }, + ); + results.total_matched + }); + }); + + // Benchmark with small result set + group.bench_function("normal_order_small", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_controller), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 10, + }, + }, + ); + results.total_matched + }); + }); + + group.bench_function("reverse_order_small", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed_controller), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 10, + }, + }, + ); + results.total_matched + }); + }); + + group.finish(); +} + +/// Benchmark pagination: first page vs deep page +fn bench_pagination_performance(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping pagination benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("pagination"); + group.sample_size(100); + + let query = "mod"; + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let page_size = 40; + + // Benchmark first page (uses partial sort optimization) + group.bench_function("page_0_size_40", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: page_size, + }, + }, + ); + results.total_matched + }); + }); + + // Benchmark 10th page (requires full sort, no optimization) + group.bench_function("page_10_size_40", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 10, + limit: page_size, + }, + }, + ); + results.total_matched + }); + }); + + // Benchmark 50th page (even deeper pagination) + group.bench_function("page_50_size_40", |b| { + b.iter(|| { + let results = picker.fuzzy_search( + black_box(&parsed), + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 50, + limit: page_size, + }, + }, + ); + results.total_matched + }); + }); + + group.finish(); +} + +/// Benchmark grep search via the FilePicker public API +fn bench_grep_search(c: &mut Criterion) { + let (sp, _sf) = match setup_once() { + Ok(result) => result, + Err(e) => { + eprintln!("Skipping grep benchmarks: {}", e); + return; + } + }; + + let guard = sp.read().unwrap(); + let picker = guard.as_ref().unwrap(); + + let mut group = c.benchmark_group("grep"); + group.sample_size(50); + + let options = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 0, + smart_case: true, + file_offset: 0, + page_limit: 100, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + + let test_queries = vec![ + ("common", "struct"), + ("specific", "DEFINE_MUTEX"), + ("path_filter", "*.h mutex"), + ]; + + let grep_parser = fff::QueryParser::new(fff::GrepConfig); + + for (name, query) in &test_queries { + let parsed = grep_parser.parse(query); + + group.bench_with_input(BenchmarkId::new("grep", name), query, |b, _| { + b.iter(|| { + let result = picker.grep(black_box(&parsed), black_box(&options)); + result.matches.len() + }); + }); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_search_queries, + bench_search_thread_scaling, + bench_search_result_limits, + bench_search_scalability, + bench_search_ordering, + bench_pagination_performance, + bench_grep_search, +); + +criterion_main!(benches); diff --git a/crates/fff-nvim/benches/grep_bench.rs b/crates/fff-nvim/benches/grep_bench.rs new file mode 100644 index 0000000..6493994 --- /dev/null +++ b/crates/fff-nvim/benches/grep_bench.rs @@ -0,0 +1,236 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use fff::file_picker::{FFFMode, FilePicker}; +use fff::{ + FilePickerOptions, GrepMode, GrepSearchOptions, SharedFilePicker, SharedFrecency, + parse_grep_query, +}; +use std::sync::OnceLock; +use std::time::Duration; + +struct TestData { + shared_picker: SharedFilePicker, +} + +static SETUP: OnceLock = OnceLock::new(); +static SETUP_NO_INDEX: OnceLock = OnceLock::new(); + +fn big_repo_path() -> String { + if let Some(path) = std::env::var_os("BIG_REPO_PATH") { + return path.to_string_lossy().into_owned(); + } + + let candidates = ["./big-repo", "../../big-repo"]; + for p in &candidates { + if std::path::Path::new(p).exists() { + return p.to_string(); + } + } + panic!( + "./big-repo not found. Run from workspace root:\n \ + git clone --depth 1 https://github.com/torvalds/linux.git big-repo" + ); +} + +fn setup() -> &'static TestData { + SETUP.get_or_init(|| { + let path = big_repo_path(); + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + eprintln!("Initializing FilePicker for {:?}...", path); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: path, + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .expect("create picker"); + + eprintln!("Waiting for scan completion..."); + shared_picker.wait_for_scan(Duration::from_secs(120)); + + eprintln!("Waiting for warmup (bigram index)..."); + loop { + let guard = shared_picker.read().expect("read lock"); + let picker = guard.as_ref().expect("picker present"); + let progress = picker.get_scan_progress(); + if progress.is_warmup_complete { + let file_count = picker.get_files().len(); + eprintln!("Ready: {} files indexed, bigram built", file_count); + break; + } + drop(guard); + std::thread::sleep(Duration::from_millis(100)); + } + + TestData { shared_picker } + }) +} + +/// Persistent picker with the bigram index disabled — every grep scans all +/// candidate files. Isolates raw scan throughput from bigram prefilter wins. +fn setup_no_index() -> &'static TestData { + SETUP_NO_INDEX.get_or_init(|| { + let path = big_repo_path(); + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + eprintln!("Initializing FilePicker (no bigram) for {:?}...", path); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + FilePickerOptions { + base_path: path, + enable_mmap_cache: false, + enable_content_indexing: false, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("create picker"); + + shared_picker.wait_for_scan(Duration::from_secs(120)); + let file_count = { + let guard = shared_picker.read().expect("read lock"); + guard.as_ref().expect("picker present").get_files().len() + }; + eprintln!("Ready (no bigram): {} files indexed", file_count); + + TestData { shared_picker } + }) +} + +fn plain_options() -> GrepSearchOptions { + GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 50, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + ..Default::default() + } +} + +fn fuzzy_options() -> GrepSearchOptions { + GrepSearchOptions { + mode: GrepMode::Fuzzy, + ..plain_options() + } +} + +/// One query per selectivity bucket: single-char, common, medium, rare, +/// multi-word, path-constrained. +const PLAIN_QUERIES: &[(&str, &str)] = &[ + ("single_char_x", "x"), + ("common_return", "return"), + ("func_mutex_lock", "mutex_lock"), + ("rare_phylink_ethtool", "phylink_ethtool"), + ("long_static_int_init", "static int __init"), + ("path_printk_c", "printk *.c"), +]; + +/// Fuzzy is expensive (>1s/iter even on warm). Keep three: exact, typo, abbrev. +const FUZZY_QUERIES: &[(&str, &str)] = &[ + ("exact_mutex_lock", "mutex_lock"), + ("typo_mutx_lock", "mutx_lock"), + ("abbrev_sched_rt", "sched_rt"), +]; + +fn bench_plain_warm(c: &mut Criterion) { + let data = setup(); + let opts = plain_options(); + + let mut group = c.benchmark_group("plain_warm"); + group.sample_size(15); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(3)); + + for (name, query) in PLAIN_QUERIES { + group.bench_with_input(BenchmarkId::from_parameter(name), query, |b, q| { + let guard = data.shared_picker.read().expect("read lock"); + let picker = guard.as_ref().expect("picker present"); + b.iter(|| { + let parsed = parse_grep_query(q); + black_box(picker.grep(&parsed, &opts)) + }); + }); + } + + group.finish(); +} + +fn bench_fuzzy_warm(c: &mut Criterion) { + let data = setup(); + let opts = fuzzy_options(); + + let mut group = c.benchmark_group("fuzzy_warm"); + // Fuzzy iters cost >1s; small sample + tight window keeps the suite fast. + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(5)); + + for (name, query) in FUZZY_QUERIES { + group.bench_with_input(BenchmarkId::from_parameter(name), query, |b, q| { + let guard = data.shared_picker.read().expect("read lock"); + let picker = guard.as_ref().expect("picker present"); + b.iter(|| { + let parsed = parse_grep_query(q); + black_box(picker.grep(&parsed, &opts)) + }); + }); + } + + group.finish(); +} + +/// `bench_plain_warm` with the bigram index off. Side-by-side with the warm +/// group it shows the per-query bigram-prefilter contribution. +fn bench_plain_no_index(c: &mut Criterion) { + let data = setup_no_index(); + let opts = plain_options(); + + // Common + medium only; rare queries cost 10-15s/iter without bigram. + let queries: &[(&str, &str)] = &[ + ("common_return", "return"), + ("func_mutex_lock", "mutex_lock"), + ]; + + let mut group = c.benchmark_group("plain_no_index"); + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(5)); + + for (name, query) in queries { + group.bench_with_input(BenchmarkId::from_parameter(name), query, |b, q| { + let guard = data.shared_picker.read().expect("read lock"); + let picker = guard.as_ref().expect("picker present"); + b.iter(|| { + let parsed = parse_grep_query(q); + black_box(picker.grep(&parsed, &opts)) + }); + }); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_plain_warm, + bench_fuzzy_warm, + bench_plain_no_index, +); + +criterion_main!(benches); diff --git a/crates/fff-nvim/benches/query_tracker_bench.rs b/crates/fff-nvim/benches/query_tracker_bench.rs new file mode 100644 index 0000000..d56e699 --- /dev/null +++ b/crates/fff-nvim/benches/query_tracker_bench.rs @@ -0,0 +1,224 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use fff::query_tracker::QueryTracker; +use rand::distributions::Alphanumeric; +use rand::prelude::*; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn generate_random_string(len: usize) -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} + +// Test data structure for benchmarks +struct TestQueryEntry { + query: String, + project_path: PathBuf, + file_path: PathBuf, + open_count: u32, + last_opened: u64, +} + +fn generate_test_data(num_entries: usize) -> Vec { + let mut rng = thread_rng(); + let mut entries = Vec::with_capacity(num_entries); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + // Generate some common queries that will be reused + let common_queries = vec![ + "main", + "test", + "config", + "utils", + "lib", + "mod", + "index", + "init", + "server", + "client", + "api", + "service", + "controller", + "model", + "view", + "component", + "handler", + "middleware", + "router", + "database", + "auth", + ]; + + // Generate some common project paths + let project_paths = vec![ + "/home/user/project1", + "/home/user/project2", + "/home/user/web-app", + "/home/user/cli-tool", + "/home/user/library", + ]; + + for _ in 0..num_entries { + let query = if rng.gen_bool(0.7) { + // 70% chance to use common query + common_queries.choose(&mut rng).unwrap().to_string() + } else { + // 30% chance to use random query + generate_random_string(rng.gen_range(3..15)) + }; + + let project_path = project_paths.choose(&mut rng).unwrap(); + let file_name = format!( + "{}.{}", + generate_random_string(rng.gen_range(5..20)), + if rng.gen_bool(0.5) { "rs" } else { "js" } + ); + let file_path = PathBuf::from(format!("{}/src/{}", project_path, file_name)); + + let entry = TestQueryEntry { + query: query.into(), + project_path: PathBuf::from(project_path), + file_path, + open_count: rng.gen_range(1..10), + last_opened: now - rng.gen_range(0..30 * 24 * 3600), // Random time within last 30 days + }; + + entries.push(entry); + } + + entries +} + +fn setup_tracker_with_data(entries: &[TestQueryEntry]) -> (QueryTracker, PathBuf) { + use std::time::{SystemTime, UNIX_EPOCH}; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let temp_dir = + std::env::temp_dir().join(format!("fff_bench_{}_{}", timestamp, rand::random::())); + let mut tracker = QueryTracker::open(temp_dir.to_str().unwrap()).unwrap(); + + // Insert all test data + for entry in entries { + for _ in 0..entry.open_count { + tracker + .track_query_completion(&entry.query, &entry.project_path, &entry.file_path) + .unwrap(); + } + } + + (tracker, temp_dir) +} + +fn cleanup_tracker_dir(dir: PathBuf) { + if dir.exists() { + let _ = std::fs::remove_dir_all(dir); + } +} + +fn bench_track_query_completion(c: &mut Criterion) { + let mut group = c.benchmark_group("track_query_completion"); + + for size in &[100, 1000, 10000] { + let entries = generate_test_data(*size); + let (mut tracker, temp_dir) = setup_tracker_with_data(&entries[..*size / 2]); // Pre-populate with half + + group.bench_with_input(BenchmarkId::new("entries", size), size, |b, _| { + let mut rng = thread_rng(); + b.iter(|| { + let entry = entries.choose(&mut rng).unwrap(); + black_box( + tracker + .track_query_completion( + black_box(&entry.query), + black_box(&entry.project_path), + black_box(&entry.file_path), + ) + .unwrap(), + ); + }); + }); + + drop(tracker); + cleanup_tracker_dir(temp_dir); + } + + group.finish(); +} + +fn bench_realistic_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("realistic_workload"); + + for size in &[1000, 10000] { + let entries = generate_test_data(*size); + let (mut tracker, temp_dir) = setup_tracker_with_data(&entries); + + group.bench_with_input(BenchmarkId::new("mixed_operations", size), size, |b, _| { + let mut rng = thread_rng(); + b.iter(|| { + let entry = entries.choose(&mut rng).unwrap(); + + // Simulate realistic usage: 70% lookups, 25% tracking, 5% history + match rng.gen_range(0..100) { + 0..70 => { + // Query entry lookup (most common operation) + let entry_result = black_box( + tracker + .get_last_query_entry( + black_box(&entry.query), + black_box(&entry.project_path), + 3, + ) + .unwrap(), + ); + black_box(entry_result); + } + 70..95 => { + // Track completion (when user opens file) + black_box( + tracker + .track_query_completion( + black_box(&entry.query), + black_box(&entry.project_path), + black_box(&entry.file_path), + ) + .unwrap(), + ); + } + 95..100 => { + // Get historical query (least common) + let history = black_box( + tracker + .get_historical_query(black_box(&entry.project_path), black_box(5)) + .unwrap(), + ); + black_box(history); + } + _ => unreachable!(), + } + }); + }); + + drop(tracker); + cleanup_tracker_dir(temp_dir); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_track_query_completion, + // Commented out - methods removed/changed in refactor: + // bench_get_query_boost, + // bench_cleanup_old_entries, + bench_realistic_workload +); +criterion_main!(benches); diff --git a/crates/fff-nvim/benches/scan_bench.rs b/crates/fff-nvim/benches/scan_bench.rs new file mode 100644 index 0000000..2b93518 --- /dev/null +++ b/crates/fff-nvim/benches/scan_bench.rs @@ -0,0 +1,220 @@ +//! Benchmarks capturing performance of index building and fs walking +use criterion::{Criterion, criterion_group, criterion_main}; +use fff::file_picker::{FFFMode, FilePicker}; +use fff::{FilePickerOptions, SharedFilePicker, SharedFrecency}; +use std::path::PathBuf; +use std::sync::Once; +use std::time::{Duration, Instant}; + +const WAIT_TIMEOUT: Duration = Duration::from_secs(300); + +static TRACING_INIT: Once = Once::new(); + +fn init_tracing() { + TRACING_INIT.call_once(|| { + use tracing_subscriber::EnvFilter; + use tracing_subscriber::fmt::format::FmtSpan; + + let _ = tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("warn,fff_search=info")), + ) + .with_span_events(FmtSpan::CLOSE) + .with_target(true) + .with_writer(std::io::stderr) + .try_init(); + }); +} + +fn resolve_repo() -> Option { + if let Ok(env_path) = std::env::var("FFF_BENCH_REPO") { + let p = PathBuf::from(env_path); + if p.exists() { + return fff::path_utils::canonicalize(&p).ok(); + } + } + // Resolve relative to the workspace root (two levels up from this crate). + let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let default = workspace_root.join("big-repo"); + if default.exists() { + return fff::path_utils::canonicalize(&default).ok(); + } + None +} + +fn init_picker( + path: &str, + content_indexing: bool, + warmup: bool, +) -> (SharedFilePicker, SharedFrecency) { + let sp = SharedFilePicker::default(); + let sf = SharedFrecency::default(); + FilePicker::new_with_shared_state( + sp.clone(), + sf.clone(), + FilePickerOptions { + base_path: path.to_string(), + enable_mmap_cache: warmup, + enable_content_indexing: content_indexing, + mode: FFFMode::Neovim, + watch: false, + ..Default::default() + }, + ) + .expect("init FilePicker"); + (sp, sf) +} + +/// Portable wait across branches (no dependency on `wait_for_indexing_complete`). +/// Polls until the scan is inactive AND the bigram index has been installed — this +/// is the end of `run_post_scan` for content-indexing builds. +fn wait_for_post_scan(sp: &SharedFilePicker, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + let ready = sp + .read() + .ok() + .and_then(|guard| { + guard + .as_ref() + .map(|p| !p.is_scan_active() && !p.is_post_scan_active()) + }) + .unwrap_or(false); + if ready { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// Portable wait for walk-only builds (content_indexing=false, no bigram). +fn wait_for_scan_done(sp: &SharedFilePicker, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + let done = sp + .read() + .ok() + .and_then(|guard| guard.as_ref().map(|p| !p.is_scan_active())) + .unwrap_or(false); + if done { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn cleanup(sp: SharedFilePicker) { + // Clean teardown: wait for scan + post-scan to finish, then drop. + // The PostScanUnsafeSnapshot holds Arc-shared data, so dropping the + // picker while post-scan threads run is memory-safe, but we wait + // for completion to avoid detached git-status threads from causing + // I/O contention on the next benchmark iteration. + sp.wait_for_indexing_complete(WAIT_TIMEOUT); + if let Ok(mut guard) = sp.write() + && let Some(mut picker) = guard.take() + { + picker.stop_background_monitor(); + } +} + +fn bench_full_init(c: &mut Criterion) { + init_tracing(); + let Some(repo) = resolve_repo() else { + eprintln!("skip: set FFF_BENCH_REPO or clone a repo to ./big-repo"); + return; + }; + let path = repo.to_string_lossy().to_string(); + eprintln!("post_scan_bench repo: {}", path); + + let mut group = c.benchmark_group("post_scan"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(120)); + + group.bench_function("full_init", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let start = Instant::now(); + let (sp, _sf) = init_picker(&path, true, true); + wait_for_post_scan(&sp, WAIT_TIMEOUT); + total += start.elapsed(); + cleanup(sp); + } + total + }); + }); + + group.finish(); +} + +fn bench_post_scan_only(c: &mut Criterion) { + init_tracing(); + let Some(repo) = resolve_repo() else { + return; + }; + let path = repo.to_string_lossy().to_string(); + + let mut group = c.benchmark_group("post_scan"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(120)); + + group.bench_function("post_scan_only", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let (sp, _sf) = init_picker(&path, true, true); + wait_for_scan_done(&sp, WAIT_TIMEOUT); + let start = Instant::now(); + wait_for_post_scan(&sp, WAIT_TIMEOUT); + total += start.elapsed(); + cleanup(sp); + } + total + }); + }); + + group.finish(); +} + +fn bench_walk_only(c: &mut Criterion) { + init_tracing(); + let Some(repo) = resolve_repo() else { + return; + }; + let path = repo.to_string_lossy().to_string(); + + let mut group = c.benchmark_group("post_scan"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(60)); + + group.bench_function("walk_only", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let start = Instant::now(); + let (sp, _sf) = init_picker(&path, false, false); + wait_for_scan_done(&sp, WAIT_TIMEOUT); + total += start.elapsed(); + cleanup(sp); + } + total + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + bench_full_init, + bench_post_scan_only, + bench_walk_only +); +criterion_main!(benches); diff --git a/crates/fff-nvim/src/bin/bench_grep_query.rs b/crates/fff-nvim/src/bin/bench_grep_query.rs new file mode 100644 index 0000000..063d081 --- /dev/null +++ b/crates/fff-nvim/src/bin/bench_grep_query.rs @@ -0,0 +1,138 @@ +/// Single-query grep benchmark with bigram index profiling. +/// +/// Usage: +/// cargo build --release --bin bench_grep_query +/// ./target/release/bench_grep_query --path ~/dev/chromium --query "MAX_FILE_SIZE" --iters 3 +/// ./target/release/bench_grep_query --path ~/dev/chromium --query "TODO" +use fff::file_picker::FilePicker; +use fff::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use std::time::Instant; + +fn fmt_dur(us: u128) -> String { + if us > 1_000_000 { + format!("{:.2}s", us as f64 / 1_000_000.0) + } else if us > 1000 { + format!("{:.2}ms", us as f64 / 1000.0) + } else { + format!("{}us", us) + } +} + +fn run_grep(picker: &FilePicker, query: &str, iters: usize) { + let options = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: usize::MAX, + mode: GrepMode::PlainText, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + + let parsed = parse_grep_query(query); + let mut times_us = Vec::with_capacity(iters); + + for i in 0..iters { + let t = Instant::now(); + let result = picker.grep(&parsed, &options); + let us = t.elapsed().as_micros(); + times_us.push(us); + + eprintln!( + " iter {}: {} ({} matches in {} files, {}/{} searched)", + i + 1, + fmt_dur(us), + result.matches.len(), + result.files_with_matches, + result.total_files_searched, + result.total_files, + ); + } + + if times_us.len() > 1 { + times_us.sort(); + let sum: u128 = times_us.iter().sum(); + let mean = sum / times_us.len() as u128; + let median = times_us[times_us.len() / 2]; + let min = times_us[0]; + let max = times_us[times_us.len() - 1]; + eprintln!( + " mean: {} median: {} min: {} max: {}", + fmt_dur(mean), + fmt_dur(median), + fmt_dur(min), + fmt_dur(max) + ); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + + let path = args + .iter() + .position(|a| a == "--path") + .and_then(|i| args.get(i + 1)) + .map(|s| s.as_str()) + .unwrap_or("."); + + let query = args + .iter() + .position(|a| a == "--query") + .and_then(|i| args.get(i + 1)) + .map(|s| s.as_str()) + .unwrap_or("TODO"); + + let iters: usize = args + .iter() + .position(|a| a == "--iters") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or(5); + + let repo = std::path::PathBuf::from(path); + if !repo.exists() { + eprintln!("Path not found: {}", path); + eprintln!("Usage: bench_grep_query --path --query [--iters N]"); + std::process::exit(1); + } + + let canonical = fff::path_utils::canonicalize(&repo).expect("Failed to canonicalize path"); + eprintln!("=== bench_grep_query ==="); + eprintln!("Path: {}", canonical.display()); + eprintln!("Query: \"{}\"", query); + eprintln!("Iters: {}", iters); + eprintln!(); + + // ── 1. Scan files ────────────────────────────────────────────────── + eprint!("[1/2] Scanning files... "); + let t = Instant::now(); + let mut picker = FilePicker::new(fff::FilePickerOptions { + base_path: canonical.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: fff::FFFMode::Neovim, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + let files = picker.get_files(); + let non_binary = files.iter().filter(|f| !f.is_binary()).count(); + eprintln!( + "{} files in {:.2}s ({} non-binary)", + files.len(), + t.elapsed().as_secs_f64(), + non_binary, + ); + + // ── 2. Grep ─────────────────────────────────────────────────────── + eprintln!( + "\n[2/2] Running grep \"{}\" x {} iterations\n", + query, iters + ); + run_grep(&picker, query, iters); +} diff --git a/crates/fff-nvim/src/bin/bench_lmdb_parallel.rs b/crates/fff-nvim/src/bin/bench_lmdb_parallel.rs new file mode 100644 index 0000000..5f283d2 --- /dev/null +++ b/crates/fff-nvim/src/bin/bench_lmdb_parallel.rs @@ -0,0 +1,150 @@ +/// Parallel LMDB contention bench. +/// +/// Spawns N child processes that all open the same frecency + query-tracker +/// dbs and hammer them. Used to measure the cost (or absence of cost) of +/// dropping the `MDB_NOLOCK | NO_SYNC | NO_META_SYNC` env flags. +/// +/// Usage: +/// cargo build --release --bin bench_lmdb_parallel +/// ./target/release/bench_lmdb_parallel --procs 4 --iters 5000 +/// +/// Env var FFF_BENCH_ROLE=worker turns the binary into a worker that talks +/// to a db path passed via FFF_BENCH_DB. +use fff::frecency::FrecencyTracker; +use fff::query_tracker::QueryTracker; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Instant; + +fn parse_args() -> (usize, usize, Option) { + let mut procs = 4usize; + let mut iters = 2000usize; + let mut db_path: Option = None; + let mut args = env::args().skip(1); + while let Some(a) = args.next() { + match a.as_str() { + "--procs" => procs = args.next().and_then(|s| s.parse().ok()).unwrap_or(procs), + "--iters" => iters = args.next().and_then(|s| s.parse().ok()).unwrap_or(iters), + "--db" => db_path = args.next(), + _ => {} + } + } + (procs, iters, db_path) +} + +fn worker_main(db: &Path, iters: usize, worker_id: u32) { + let frecency_path = db.join("frecency"); + let history_path = db.join("history"); + + let frecency = FrecencyTracker::open(&frecency_path).expect("frecency open"); + let mut query_tracker = QueryTracker::open(&history_path).expect("query tracker open"); + + let project = PathBuf::from("/bench/project"); + let started = Instant::now(); + + for i in 0..iters { + let file = PathBuf::from(format!( + "/bench/project/src/file_{}_{}.rs", + worker_id, + i % 256 + )); + // Frecency write path — same call as track_access on BufEnter. + frecency.track_access(&file).expect("frecency track_access"); + + // Query tracker write path — same as track_query_completion on file open. + let query = format!("q{}", i % 32); + query_tracker + .track_query_completion(&query, &project, &file) + .expect("query tracker track_query_completion"); + + // A read every few iterations. + if i % 8 == 0 { + let _ = frecency.seconds_since_last_access(&file).ok(); + let _ = query_tracker + .get_historical_query(&project, 0) + .ok() + .flatten(); + } + } + + let elapsed = started.elapsed(); + eprintln!( + "worker {} done: {} iters in {:?} ({:.0} ops/s)", + worker_id, + iters, + elapsed, + iters as f64 * 3.0 / elapsed.as_secs_f64() + ); +} + +fn driver_main(procs: usize, iters: usize, db_override: Option) { + let db = db_override.map(PathBuf::from).unwrap_or_else(|| { + std::env::temp_dir().join(format!("fff_bench_lmdb_{}", std::process::id())) + }); + let _ = std::fs::remove_dir_all(&db); + std::fs::create_dir_all(&db).expect("create db dir"); + + let exe = env::current_exe().expect("current_exe"); + + eprintln!( + "driver: launching {} workers, {} iters each, db={}", + procs, + iters, + db.display() + ); + + let started = Instant::now(); + let mut children = Vec::with_capacity(procs); + for worker_id in 0..procs { + let child = Command::new(&exe) + .env("FFF_BENCH_ROLE", "worker") + .env("FFF_BENCH_DB", &db) + .env("FFF_BENCH_ITERS", iters.to_string()) + .env("FFF_BENCH_WORKER_ID", worker_id.to_string()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn worker"); + children.push(child); + } + + let mut failures = 0u32; + for mut child in children { + let status = child.wait().expect("wait"); + if !status.success() { + failures += 1; + } + } + + let elapsed = started.elapsed(); + let total_ops = procs as f64 * iters as f64 * 3.0; + eprintln!( + "driver: all workers done in {:?}, ~{:.0} ops/s aggregate, failures={}", + elapsed, + total_ops / elapsed.as_secs_f64(), + failures + ); + + let _ = std::fs::remove_dir_all(&db); +} + +fn main() { + // Worker branch: spawned by driver to contend on the same db. + if env::var("FFF_BENCH_ROLE").as_deref() == Ok("worker") { + let db = env::var("FFF_BENCH_DB").expect("FFF_BENCH_DB"); + let iters: usize = env::var("FFF_BENCH_ITERS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(1000); + let worker_id: u32 = env::var("FFF_BENCH_WORKER_ID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + worker_main(Path::new(&db), iters, worker_id); + return; + } + + let (procs, iters, db) = parse_args(); + driver_main(procs, iters, db); +} diff --git a/crates/fff-nvim/src/bin/bench_search_only.rs b/crates/fff-nvim/src/bin/bench_search_only.rs new file mode 100644 index 0000000..3dc7623 --- /dev/null +++ b/crates/fff-nvim/src/bin/bench_search_only.rs @@ -0,0 +1,117 @@ +/// Simple search profiler that directly uses scan_filesystem without background thread overhead +use fff::file_picker::FilePicker; +use fff::{FuzzySearchOptions, PaginationArgs, QueryParser}; +use std::time::Instant; + +fn load_picker(path: &std::path::Path) -> FilePicker { + let mut picker = FilePicker::new(fff::FilePickerOptions { + base_path: path.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: fff::FFFMode::Neovim, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + picker +} + +fn main() { + let big_repo_path = std::path::PathBuf::from("./big-repo"); + + if !big_repo_path.exists() { + eprintln!( + "./big-repo directory does not exist. Run git clone https://github.com/torvalds/linux.git big-repo" + ); + return; + } + + let canonical_path = + fff::path_utils::canonicalize(&big_repo_path).expect("Failed to canonicalize path"); + + eprintln!("Loading files from: {:?}", canonical_path); + + let start = Instant::now(); + let picker = load_picker(&canonical_path); + eprintln!( + "✓ Loaded {} files in {:.2}s\n", + picker.get_files().len(), + start.elapsed().as_secs_f64() + ); + + // Test queries + let test_queries = vec![ + ("short_common", "mod", 500), + ("medium_specific", "controller", 200), + ("long_rare", "user_authentication", 100), + ("typo_resistant", "contrlr", 200), + ("path_like", "src/lib", 150), + ("two_char", "st", 300), + ("partial_word", "test", 200), + ("deep_path", "drivers/net", 100), + ("extension", ".rs", 200), + ]; + + eprintln!("Running search profiler..."); + eprintln!("Query | Iterations | Total Time | Avg Time | Matches"); + eprintln!("----------------------|------------|------------|-----------|--------"); + + let global_start = Instant::now(); + let mut total_iterations = 0; + + for (name, query, iterations) in test_queries { + let start = Instant::now(); + let mut match_count = 0; + + for _ in 0..iterations { + let parser = QueryParser::default(); + let parsed = parser.parse(query); + let results = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + match_count += results.total_matched; + } + + let elapsed = start.elapsed(); + let avg_time = elapsed / iterations as u32; + + eprintln!( + "{:<21} | {:>10} | {:>9.2}s | {:>7}µs | {}", + name, + iterations, + elapsed.as_secs_f64(), + avg_time.as_micros(), + match_count / iterations + ); + + total_iterations += iterations; + } + + let total_time = global_start.elapsed(); + + eprintln!("\n=== Summary ==="); + eprintln!("Total searches: {}", total_iterations); + eprintln!("Total time: {:.2}s", total_time.as_secs_f64()); + eprintln!( + "Average per search: {}µs", + (total_time.as_micros() as usize) / total_iterations + ); + eprintln!( + "Searches per sec: {:.0}", + total_iterations as f64 / total_time.as_secs_f64() + ); + eprintln!( + "\nYou can now run: perf record -g --call-graph dwarf -F 999 ./target/release/search_only" + ); +} diff --git a/crates/fff-nvim/src/bin/fuzzy_grep_test.rs b/crates/fff-nvim/src/bin/fuzzy_grep_test.rs new file mode 100644 index 0000000..e3bceac --- /dev/null +++ b/crates/fff-nvim/src/bin/fuzzy_grep_test.rs @@ -0,0 +1,183 @@ +/// Fuzzy grep quality test against ~/dev/lightsource +/// +/// Runs queries through the fuzzy grep pipeline and prints results +/// so we can verify match quality. +/// +/// Usage: +/// cargo run --release --bin fuzzy_grep_test # runs default test queries +/// cargo run --release --bin fuzzy_grep_test -- "query" # runs a single user query +use fff::file_picker::FilePicker; +use fff::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use std::path::Path; +use std::time::Instant; + +fn create_picker(path: &Path) -> FilePicker { + let mut picker = FilePicker::new(fff::FilePickerOptions { + base_path: path.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: fff::FFFMode::Neovim, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + picker +} + +fn run_fuzzy_query(picker: &FilePicker, query: &str, label: &str) { + let options = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 100, + mode: GrepMode::Fuzzy, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + + let parsed = parse_grep_query(query); + let start = Instant::now(); + let result = picker.grep(&parsed, &options); + let elapsed = start.elapsed(); + + eprintln!("══════════════════════════════════════════════════════════════"); + eprintln!(" Query: \"{}\" ({})", query, label); + eprintln!( + " Results: {} matches in {} files ({:.2}ms)", + result.matches.len(), + result.total_files_searched, + elapsed.as_secs_f64() * 1000.0, + ); + eprintln!("══════════════════════════════════════════════════════════════"); + + if result.matches.is_empty() { + eprintln!(" (no matches)\n"); + return; + } + + // Group by file for readability + let mut current_file_idx = usize::MAX; + for (i, m) in result.matches.iter().enumerate() { + if m.file_index != current_file_idx { + current_file_idx = m.file_index; + let file = &result.files[m.file_index]; + eprintln!("\n ┌─ {}", file.relative_path(picker)); + } + + // Truncate long lines for display + let display_line = if m.line_content.len() > 100 { + format!("{}...", &m.line_content[..100]) + } else { + m.line_content.clone() + }; + + let score_str = m + .fuzzy_score + .map(|s| format!("score={}", s)) + .unwrap_or_else(|| "no-score".to_string()); + + let offsets_str = if m.match_byte_offsets.is_empty() { + String::new() + } else { + // Show what text fragments are highlighted + let fragments: Vec = m + .match_byte_offsets + .iter() + .filter_map(|&(s, e)| { + m.line_content + .get(s as usize..e as usize) + .map(|frag| format!("\"{}\"", frag)) + }) + .collect(); + format!(" hl=[{}]", fragments.join(",")) + }; + + eprintln!( + " │ L{:<5} [{}{}] {}", + m.line_number, + score_str, + offsets_str, + display_line.trim(), + ); + + // Cap output at 50 lines + if i >= 49 { + let remaining = result.matches.len() - 50; + if remaining > 0 { + eprintln!(" │ ... and {} more matches", remaining); + } + break; + } + } + eprintln!(); +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + + let (repo_path, queries) = if let Some(idx) = args.iter().position(|a| a == "--path") { + let path = args + .get(idx + 1) + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + eprintln!("--path requires an argument"); + std::process::exit(1); + }); + let queries: Vec = args + .iter() + .enumerate() + .filter(|(i, _)| *i != idx && *i != idx + 1) + .map(|(_, s)| s.clone()) + .collect(); + (path, queries) + } else { + let path = std::path::PathBuf::from( + std::env::var("HOME").unwrap_or_else(|_| "/Users/neogoose".to_string()), + ) + .join("dev/lightsource"); + (path, args) + }; + + if !repo_path.exists() { + eprintln!("Repository not found at: {:?}", repo_path); + std::process::exit(1); + } + + let canonical = fff::path_utils::canonicalize(&repo_path).expect("Failed to canonicalize path"); + eprintln!("=== Fuzzy Grep Quality Test ==="); + eprintln!("Repository: {:?}\n", canonical); + + eprintln!("Loading files..."); + let load_start = Instant::now(); + let picker = create_picker(&canonical); + let files = picker.get_files(); + let non_binary = files.iter().filter(|f| !f.is_binary()).count(); + eprintln!( + "Loaded {} files ({} non-binary) in {:.2}s\n", + files.len(), + non_binary, + load_start.elapsed().as_secs_f64() + ); + + if queries.is_empty() { + // Run default test queries + run_fuzzy_query(&picker, "shcema", "transposition of 'schema'"); + run_fuzzy_query(&picker, "SortedMap", "should match SortedArrayMap"); + run_fuzzy_query( + &picker, + "struct SortedMap", + "should NOT match SourcingProjectMetadataParts", + ); + } else { + // Run user-provided queries + for query in &queries { + run_fuzzy_query(&picker, query, "user query"); + } + } + + eprintln!("=== Done ==="); +} diff --git a/crates/fff-nvim/src/bin/grep_profiler.rs b/crates/fff-nvim/src/bin/grep_profiler.rs new file mode 100644 index 0000000..4c0386b --- /dev/null +++ b/crates/fff-nvim/src/bin/grep_profiler.rs @@ -0,0 +1,453 @@ +/// Live grep benchmark profiler for fff.nvim +/// +/// Benchmarks the full grep pipeline against a large repository (Linux kernel). +/// Measures cold-cache, warm-cache, and incremental typing latencies to simulate +/// real user interaction patterns. +/// +/// Uses FilePicker::collect_files for synchronous scanning (no background thread). +/// +/// Usage: +/// cargo build --release --bin grep_profiler +/// ./target/release/grep_profiler [--path /path/to/repo] +use fff::file_picker::FilePicker; +use fff::grep::{GrepMode, GrepSearchOptions, parse_grep_query}; +use std::time::{Duration, Instant}; + +fn create_picker(path: &std::path::Path) -> FilePicker { + let mut picker = FilePicker::new(fff::FilePickerOptions { + base_path: path.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: fff::FFFMode::Neovim, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + picker +} + +struct BenchStats { + times: Vec, +} + +impl BenchStats { + fn new() -> Self { + Self { times: Vec::new() } + } + + fn push(&mut self, d: Duration) { + self.times.push(d); + } + + fn mean(&self) -> Duration { + let total: Duration = self.times.iter().sum(); + total / self.times.len() as u32 + } + + fn median(&self) -> Duration { + let mut sorted = self.times.clone(); + sorted.sort(); + sorted[sorted.len() / 2] + } + + fn p95(&self) -> Duration { + let mut sorted = self.times.clone(); + sorted.sort(); + let idx = ((sorted.len() as f64) * 0.95) as usize; + sorted[idx.min(sorted.len() - 1)] + } + + fn p99(&self) -> Duration { + let mut sorted = self.times.clone(); + sorted.sort(); + let idx = ((sorted.len() as f64) * 0.99) as usize; + sorted[idx.min(sorted.len() - 1)] + } + + fn min(&self) -> Duration { + *self.times.iter().min().unwrap() + } + + fn max(&self) -> Duration { + *self.times.iter().max().unwrap() + } +} + +struct GrepBench<'a> { + picker: &'a FilePicker, + options: GrepSearchOptions, +} + +impl<'a> GrepBench<'a> { + fn new(picker: &'a FilePicker) -> Self { + Self::with_mode(picker, GrepMode::PlainText) + } + + fn with_mode(picker: &'a FilePicker, mode: GrepMode) -> Self { + Self { + picker, + options: GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 50, + mode, + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }, + } + } + + /// Run a single grep search, return (duration, match_count, files_searched) + fn run_once(&self, query: &str) -> (Duration, usize, usize) { + let parsed = parse_grep_query(query); + let start = Instant::now(); + let result = self.picker.grep(&parsed, &self.options); + let elapsed = start.elapsed(); + (elapsed, result.matches.len(), result.total_files_searched) + } + + /// Benchmark a query with multiple iterations + fn bench_query(&self, query: &str, iterations: usize) -> (BenchStats, usize, usize) { + let mut stats = BenchStats::new(); + let mut last_matches = 0; + let mut last_files_searched = 0; + + for _ in 0..iterations { + let (elapsed, matches, files_searched) = self.run_once(query); + stats.push(elapsed); + last_matches = matches; + last_files_searched = files_searched; + } + + (stats, last_matches, last_files_searched) + } +} + +fn fmt_dur(d: Duration) -> String { + let us = d.as_micros(); + if us > 1_000_000 { + format!("{:.2}s", d.as_secs_f64()) + } else if us > 1000 { + format!("{:.2}ms", us as f64 / 1000.0) + } else { + format!("{}us", us) + } +} + +fn print_row(name: &str, stats: &BenchStats, matches: usize, files_searched: usize, iters: usize) { + eprintln!( + " {:<24} | {:>8} | {:>8} | {:>8} | {:>8} | {:>8} | {:>8} | {:>6} | {:>6} | {:>4}", + name, + fmt_dur(stats.mean()), + fmt_dur(stats.median()), + fmt_dur(stats.p95()), + fmt_dur(stats.p99()), + fmt_dur(stats.min()), + fmt_dur(stats.max()), + matches, + files_searched, + iters, + ); +} + +fn print_header() { + eprintln!( + " {:<24} | {:>8} | {:>8} | {:>8} | {:>8} | {:>8} | {:>8} | {:>6} | {:>6} | {:>4}", + "Name", "Mean", "Median", "P95", "P99", "Min", "Max", "Match", "Files", "Iter" + ); + eprintln!( + " {:-<24}-+-{:-<8}-+-{:-<8}-+-{:-<8}-+-{:-<8}-+-{:-<8}-+-{:-<8}-+-{:-<6}-+-{:-<6}-+-{:-<4}", + "", "", "", "", "", "", "", "", "", "" + ); +} + +fn main() { + // Parse args + let args: Vec = std::env::args().collect(); + let repo_path = if let Some(idx) = args.iter().position(|a| a == "--path") { + args.get(idx + 1) + .map(|s| s.as_str()) + .unwrap_or("./big-repo") + } else { + "./big-repo" + }; + + let repo = std::path::PathBuf::from(repo_path); + if !repo.exists() { + eprintln!("Repository not found at: {}", repo_path); + eprintln!("Usage: grep_profiler [--path /path/to/large/repo]"); + std::process::exit(1); + } + + let canonical = fff::path_utils::canonicalize(&repo).expect("Failed to canonicalize path"); + eprintln!("=== FFF Live Grep Profiler ==="); + eprintln!("Repository: {:?}", canonical); + + // Direct file loading (no background thread) + eprintln!("\n[1/7] Loading files..."); + let load_start = Instant::now(); + let picker = create_picker(&canonical); + let load_time = load_start.elapsed(); + let files = picker.get_files(); + let non_binary = files.iter().filter(|f| !f.is_binary()).count(); + let large_files = files.iter().filter(|f| f.size > 10 * 1024 * 1024).count(); + eprintln!( + " Loaded {} files in {:.2}s ({} non-binary, {} >10MB skipped)\n", + files.len(), + load_time.as_secs_f64(), + non_binary, + large_files, + ); + + let bench = GrepBench::new(&picker); + + eprintln!("[2/7] Cold cache benchmarks (first search, mmap not yet loaded)"); + eprintln!(" Each query runs once with fresh FileItem mmaps.\n"); + print_header(); + + let cold_queries: Vec<(&str, &str)> = vec![ + ("cold_common_2char", "if"), + ("cold_common_word", "return"), + ("cold_specific_func", "mutex_lock"), + ("cold_struct_name", "inode_operations"), + ("cold_define", "MODULE_LICENSE"), + ("cold_rare_string", "phylink_ethtool"), + ("cold_path_filter", "printk *.c"), + ("cold_long_query", "static int __init"), + ]; + + for (name, query) in &cold_queries { + // Re-load files to get fresh FileItems with no cached mmaps + let fresh_picker = create_picker(&canonical); + let fresh_bench = GrepBench::new(&fresh_picker); + let (stats, matches, files_searched) = fresh_bench.bench_query(query, 1); + print_row(name, &stats, matches, files_searched, 1); + } + + eprintln!("\n[3/7] Warm cache benchmarks (plain text, mmap cache populated)"); + eprintln!(" Running 3 warmup iterations, then measuring.\n"); + print_header(); + + let warm_queries: Vec<(&str, &str, usize)> = vec![ + ("warm_2char", "if", 10), + ("warm_common_word", "return", 10), + ("warm_function_call", "mutex_lock", 15), + ("warm_struct_name", "inode_operations", 15), + ("warm_define", "MODULE_LICENSE", 15), + ("warm_rare_string", "phylink_ethtool", 20), + ("warm_include", "#include", 10), + ("warm_comment", "TODO", 15), + ("warm_type_decl", "struct file", 15), + ("warm_error_path", "err = -EINVAL", 15), + ("warm_long_pattern", "static int __init", 15), + ("warm_very_common", "int", 10), + ("warm_single_char", "x", 10), + ("warm_path_constraint", "printk *.c", 15), + ("warm_dir_constraint", "mutex /kernel/", 15), + ]; + + // Warmup pass - populate mmap cache + for (_, query, _) in &warm_queries { + for _ in 0..3 { + bench.run_once(query); + } + } + + for (name, query, iters) in &warm_queries { + let (stats, matches, files_searched) = bench.bench_query(query, *iters); + print_row(name, &stats, matches, files_searched, *iters); + } + + // Bigram-related benchmarks are omitted: the bigram index is built + // asynchronously by the picker and is exercised through picker.grep(). + + // ── Fuzzy grep benchmarks ───────────────────────────────────────────── + eprintln!("\n[4/7] Fuzzy grep warm benchmarks"); + eprintln!(" Running 3 warmup iterations, then measuring.\n"); + print_header(); + + let fuzzy_bench = GrepBench::with_mode(&picker, GrepMode::Fuzzy); + + let fuzzy_queries: Vec<(&str, &str, usize)> = vec![ + ("fuzzy_exact", "mutex_lock", 15), + ("fuzzy_typo", "mutx_lock", 15), + ("fuzzy_camel", "InodeOps", 15), + ("fuzzy_abbrev", "sched_rt", 15), + ("fuzzy_short", "kfr", 15), + ("fuzzy_common", "return", 10), + ("fuzzy_define", "MODULE_LICENSE", 15), + ("fuzzy_struct", "file_operations", 15), + ("fuzzy_long", "static_int_init", 15), + ("fuzzy_path", "printk *.c", 15), + ]; + + // Warmup + for (_, query, _) in &fuzzy_queries { + for _ in 0..3 { + fuzzy_bench.run_once(query); + } + } + + for (name, query, iters) in &fuzzy_queries { + let (stats, matches, files_searched) = fuzzy_bench.bench_query(query, *iters); + print_row(name, &stats, matches, files_searched, *iters); + } + + // ── Fuzzy incremental typing ──────────────────────────────────────── + eprintln!("\n[5/7] Fuzzy incremental typing simulation"); + eprintln!(" Simulates user typing character by character (fuzzy mode).\n"); + + let fuzzy_typing_sequences: Vec<(&str, Vec<&str>)> = vec![ + ( + "mutex_lock", + vec![ + "m", + "mu", + "mut", + "mute", + "mutex", + "mutex_", + "mutex_l", + "mutex_lo", + "mutex_loc", + "mutex_lock", + ], + ), + ("printk", vec!["p", "pr", "pri", "prin", "print", "printk"]), + ("kfree", vec!["k", "kf", "kfr", "kfre", "kfree"]), + ]; + + for (name, sequence) in &fuzzy_typing_sequences { + eprintln!(" Typing '{}' ({} keystrokes):", name, sequence.len()); + eprintln!( + " {:>16} | {:>8} | {:>6} | {:>6}", + "Query", "Latency", "Match", "Files" + ); + eprintln!(" {:-<16}-+-{:-<8}-+-{:-<6}-+-{:-<6}", "", "", "", ""); + + for prefix in sequence { + let (elapsed, matches, files_searched) = fuzzy_bench.run_once(prefix); + eprintln!( + " {:>16} | {:>8} | {:>6} | {:>6}", + format!("\"{}\"", prefix), + fmt_dur(elapsed), + matches, + files_searched, + ); + } + eprintln!(); + } + + eprintln!("[6/7] Incremental typing simulation (plain text)"); + eprintln!(" Simulates user typing character by character.\n"); + + let bench = GrepBench::new(&picker); + let typing_sequences: Vec<(&str, Vec<&str>)> = vec![ + ( + "mutex_lock", + vec![ + "m", + "mu", + "mut", + "mute", + "mutex", + "mutex_", + "mutex_l", + "mutex_lo", + "mutex_loc", + "mutex_lock", + ], + ), + ("printk", vec!["p", "pr", "pri", "prin", "print", "printk"]), + ("inode", vec!["i", "in", "ino", "inod", "inode"]), + ("kfree", vec!["k", "kf", "kfr", "kfre", "kfree"]), + ]; + + for (name, sequence) in &typing_sequences { + eprintln!(" Typing '{}' ({} keystrokes):", name, sequence.len()); + eprintln!( + " {:>16} | {:>8} | {:>6} | {:>6}", + "Query", "Latency", "Match", "Files" + ); + eprintln!(" {:-<16}-+-{:-<8}-+-{:-<6}-+-{:-<6}", "", "", "", ""); + + for prefix in sequence { + let (elapsed, matches, files_searched) = bench.run_once(prefix); + eprintln!( + " {:>16} | {:>8} | {:>6} | {:>6}", + format!("\"{}\"", prefix), + fmt_dur(elapsed), + matches, + files_searched, + ); + } + eprintln!(); + } + + eprintln!("[7/7] Pagination benchmark"); + eprintln!(" Testing page_offset performance for common query.\n"); + + let pagination_query = "return"; + eprintln!(" Query: \"{}\"", pagination_query); + eprintln!( + " {:>6} | {:>12} | {:>8} | {:>6} | {:>12}", + "Page", "File offset", "Latency", "Matches", "Next offset" + ); + eprintln!( + " {:-<6}-+-{:-<12}-+-{:-<8}-+-{:-<6}-+-{:-<12}", + "", "", "", "", "" + ); + + let mut file_offset = 0usize; + for page in 0..10 { + let parsed = parse_grep_query(pagination_query); + let opts = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset, + page_limit: 50, + mode: Default::default(), + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let start = Instant::now(); + let result = picker.grep(&parsed, &opts); + + let elapsed = start.elapsed(); + eprintln!( + " {:>6} | {:>12} | {:>8} | {:>6} | {:>12}", + page, + file_offset, + fmt_dur(elapsed), + result.matches.len(), + result.next_file_offset, + ); + + if result.next_file_offset == 0 || result.matches.is_empty() { + eprintln!(" (no more results)"); + break; + } + file_offset = result.next_file_offset; + } + + eprintln!("\n=== Summary ==="); + eprintln!(" Total indexed files: {}", files.len()); + eprintln!(" Non-binary files: {}", non_binary); + eprintln!(" Files > 10MB (skipped): {}", large_files); + + std::thread::sleep(Duration::from_millis(100)); + + eprintln!("\nDone. For perf profiling:"); + eprintln!(" perf record -g --call-graph dwarf -F 999 ./target/release/grep_profiler"); + eprintln!(" perf report --no-children"); +} diff --git a/crates/fff-nvim/src/bin/grep_vs_rg.rs b/crates/fff-nvim/src/bin/grep_vs_rg.rs new file mode 100644 index 0000000..f202fc1 --- /dev/null +++ b/crates/fff-nvim/src/bin/grep_vs_rg.rs @@ -0,0 +1,443 @@ +use fff::file_picker::FilePicker; +/// FFF vs ripgrep comparison benchmark +/// +/// Demonstrates why a persistent in-process search engine (fff) is fundamentally +/// faster than shelling out to ripgrep on every keystroke (telescope/fzf-lua). +/// +/// Each query is run N iterations to show the real-world advantage: +/// - fff: pre-indexed files + cached mmaps = near-zero overhead per search +/// - rg: fork/exec + directory traversal + gitignore parsing + file opens per invocation +/// +/// Sections: +/// 1. Raw engine speed — fff count-only vs rg --count-matches (N iterations) +/// 2. Full results — fff collect-all vs rg full line output (N iterations) +/// 3. First-page — fff paginated (50 results) vs rg telescope-style +/// (spawn, stream 50 lines, kill) — the real UI scenario (N iterations) +/// +/// The rg commands use telescope's default vimgrep_arguments: +/// rg --color=never --no-heading --with-filename --line-number --column --smart-case +/// +/// Usage: +/// cargo build --release --bin grep_vs_rg +/// ./target/release/grep_vs_rg [--path /path/to/repo] [--iters 5] +use fff::grep::{GrepSearchOptions, parse_grep_query}; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; + +/// Number of times each query is repeated (overridable with --iters). +const DEFAULT_ITERS: usize = 5; + +fn create_picker(path: &Path) -> FilePicker { + let mut picker = FilePicker::new(fff::FilePickerOptions { + base_path: path.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: fff::FFFMode::Neovim, + ..Default::default() + }) + .expect("Failed to create FilePicker"); + picker.collect_files().expect("Failed to collect files"); + picker +} + +/// Telescope's default vimgrep_arguments applied to any rg command. +/// Also limits rg's thread count to match rayon's pool (fair comparison). +fn apply_telescope_args(cmd: &mut Command, threads: usize) { + cmd.arg("--color=never") + .arg("--no-heading") + .arg("--with-filename") + .arg("--line-number") + .arg("--column") + .arg("--smart-case") + .arg("--fixed-strings") + .arg("--max-filesize") + .arg("10M") + .arg("--threads") + .arg(threads.to_string()); +} + +/// Run ripgrep counting matches via --count-matches. +fn run_rg_count( + repo_path: &Path, + pattern: &str, + case_insensitive: bool, + threads: usize, +) -> (usize, Duration) { + let start = Instant::now(); + let mut cmd = Command::new("rg"); + cmd.arg("--count-matches").arg("--no-filename"); + apply_telescope_args(&mut cmd, threads); + if case_insensitive { + cmd.arg("--ignore-case"); + } + cmd.arg(pattern).current_dir(repo_path); + + let output = cmd.output().expect("Failed to run rg"); + let elapsed = start.elapsed(); + let stdout = String::from_utf8_lossy(&output.stdout); + let count: usize = stdout + .lines() + .filter_map(|l| l.trim().parse::().ok()) + .sum(); + (count, elapsed) +} + +/// Run ripgrep collecting full line output. +fn run_rg_lines( + repo_path: &Path, + pattern: &str, + case_insensitive: bool, + threads: usize, +) -> (usize, Duration) { + let start = Instant::now(); + let mut cmd = Command::new("rg"); + apply_telescope_args(&mut cmd, threads); + if case_insensitive { + cmd.arg("--ignore-case"); + } + cmd.arg(pattern).current_dir(repo_path); + + let output = cmd.output().expect("Failed to run rg"); + let elapsed = start.elapsed(); + let count = bytecount(&output.stdout, b'\n'); + (count, elapsed) +} + +/// Run ripgrep the way telescope/fzf-lua actually do it: spawn rg as a +/// streaming subprocess, read stdout line-by-line, and kill the process +/// after `limit` lines. This is the realistic "first page" scenario. +fn run_rg_page( + repo_path: &Path, + pattern: &str, + case_insensitive: bool, + limit: usize, + threads: usize, +) -> (usize, Duration) { + use std::io::{BufRead, BufReader}; + use std::process::Stdio; + + let start = Instant::now(); + let mut rg_cmd = Command::new("rg"); + apply_telescope_args(&mut rg_cmd, threads); + if case_insensitive { + rg_cmd.arg("--ignore-case"); + } + rg_cmd + .arg(pattern) + .current_dir(repo_path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + + let mut child = rg_cmd.spawn().expect("Failed to spawn rg"); + let stdout = child.stdout.take().expect("Failed to get rg stdout"); + let reader = BufReader::new(stdout); + + let mut count = 0; + for _line in reader.lines() { + if _line.is_err() { + break; + } + count += 1; + if count >= limit { + break; + } + } + + // Kill rg immediately --- this is what telescope does when the picker + // closes or the query changes (plenary.job:shutdown). + let _ = child.kill(); + let _ = child.wait(); + let elapsed = start.elapsed(); + + (count, elapsed) +} + +fn bytecount(bytes: &[u8], needle: u8) -> usize { + bytes.iter().filter(|&&b| b == needle).count() +} + +/// fff full: collects all GrepMatch structs (what the UI uses). +fn run_fff_full(picker: &FilePicker, query: &str) -> (usize, Duration) { + let parsed = parse_grep_query(query); + let options = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: usize::MAX, + smart_case: true, + file_offset: 0, + page_limit: usize::MAX, + mode: Default::default(), + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let start = Instant::now(); + let result = picker.grep(&parsed, &options); + let elapsed = start.elapsed(); + (result.matches.len(), elapsed) +} + +/// fff paginated: first 50 results only (real UI scenario). +fn run_fff_page(picker: &FilePicker, query: &str) -> (usize, Duration) { + let parsed = parse_grep_query(query); + let options = GrepSearchOptions { + max_file_size: 10 * 1024 * 1024, + max_matches_per_file: 200, + smart_case: true, + file_offset: 0, + page_limit: 50, + mode: Default::default(), + time_budget_ms: 0, + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: false, + abort_signal: None, + }; + let start = Instant::now(); + let result = picker.grep(&parsed, &options); + let elapsed = start.elapsed(); + (result.matches.len(), elapsed) +} + +#[allow(dead_code)] +struct IterStats { + min: Duration, + avg: Duration, + count: usize, +} + +fn run_n (usize, Duration)>(f: F, n: usize) -> IterStats { + let mut times = Vec::with_capacity(n); + let mut count = 0; + for _ in 0..n { + let (c, d) = f(); + count = c; + times.push(d); + } + times.sort(); + let min = times[0]; + let avg = times.iter().sum::() / n as u32; + IterStats { min, avg, count } +} + +fn fmt_dur(d: Duration) -> String { + let us = d.as_micros(); + if us > 1_000_000 { + format!("{:.2}s", d.as_secs_f64()) + } else if us > 1000 { + format!("{:.1}ms", us as f64 / 1000.0) + } else { + format!("{}us", us) + } +} + +fn ratio_str(a: Duration, b: Duration) -> String { + if a.is_zero() || b.is_zero() { + return "-".to_string(); + } + let r = b.as_secs_f64() / a.as_secs_f64(); + format!("{:.1}x", r) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let repo_path = if let Some(idx) = args.iter().position(|a| a == "--path") { + args.get(idx + 1) + .map(|s| s.as_str()) + .unwrap_or("./big-repo") + } else { + "./big-repo" + }; + let iters = if let Some(idx) = args.iter().position(|a| a == "--iters") { + args.get(idx + 1) + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_ITERS) + } else { + DEFAULT_ITERS + }; + + let repo = std::path::PathBuf::from(repo_path); + if !repo.exists() { + eprintln!("Repository not found at: {}", repo_path); + std::process::exit(1); + } + + let canonical = fff::path_utils::canonicalize(&repo).expect("Failed to canonicalize path"); + + let rg_version = Command::new("rg") + .arg("--version") + .output() + .expect("ripgrep (rg) not found in PATH"); + let rg_ver = String::from_utf8_lossy(&rg_version.stdout); + + // Match rg's thread count to rayon's (both default to logical CPU count). + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + eprintln!("=== FFF vs ripgrep (telescope-style) ==="); + eprintln!("Repo: {:?}", canonical); + eprintln!("rg: {}", rg_ver.lines().next().unwrap_or("?")); + eprintln!("Threads: {} (rg -j{} = rayon default)", threads, threads); + eprintln!("Iterations: {} per query", iters); + eprintln!(); + + eprintln!("[1/5] Indexing files..."); + let picker = create_picker(&canonical); + let files = picker.get_files(); + let non_binary = files.iter().filter(|f| !f.is_binary()).count(); + eprintln!(" {} files ({} searchable)\n", files.len(), non_binary); + + eprintln!("[2/5] Warming caches (fff mmap + OS page cache)..."); + for q in &["return", "mutex", "struct", "include", "if", "int"] { + let _ = run_fff_page(&picker, q); + let _ = run_rg_count(&canonical, q, true, threads); + } + eprintln!(" mmap cache: warmed\n"); + + // (name, query, case_insensitive_for_rg) + let queries: Vec<(&str, &str, bool)> = vec![ + ("single_char", "x", true), + ("short_common", "if", true), + ("very_common", "int", true), + ("common_keyword", "return", true), + ("preprocessor", "#include", true), + ("function_call", "mutex_lock", true), + ("multi_word", "static int __init", true), + ("type_decl", "struct file", true), + ("macro_define", "MODULE_LICENSE", false), + ("kernel_api", "EXPORT_SYMBOL", false), + ("error_path", "err = -EINVAL", false), + ("comment_tag", "TODO", false), + ("struct_name", "inode_operations", true), + ("rare_symbol", "phylink_ethtool", true), + ("long_literal", "This program is free software", true), + ]; + + eprintln!( + "\n[4/5] Full results: fff (collect all) vs rg (full line output) ({} iters, showing min)\n", + iters + ); + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + "Query", "fff min", "count", "rg min", "count", "fff/rg" + ); + eprintln!( + " {:-<22}-+-{:-<9}-{:-<10}-+-{:-<9}-{:-<10}-+-{:-<7}", + "", "", "", "", "", "" + ); + + let mut fff_full_total = Duration::ZERO; + let mut rg_full_total = Duration::ZERO; + + for (name, query, ci) in &queries { + let q = *query; + let ci = *ci; + let fs = run_n(|| run_fff_full(&picker, q), iters); + let rs = run_n(|| run_rg_lines(&canonical, q, ci, threads), iters); + + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + name, + fmt_dur(fs.min), + fs.count, + fmt_dur(rs.min), + rs.count, + ratio_str(fs.min, rs.min), + ); + + fff_full_total += fs.min; + rg_full_total += rs.min; + } + + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + "TOTAL", + fmt_dur(fff_full_total), + "", + fmt_dur(rg_full_total), + "", + ratio_str(fff_full_total, rg_full_total), + ); + + eprintln!( + "\n[5/5] First-page latency --- the real UI scenario ({} iters, showing min)", + iters + ); + eprintln!(" fff: paginated search (50 matches) from warm mmap cache"); + eprintln!(" rg: telescope-style (spawn, stream 50 lines, kill) --- per-keystroke cost\n"); + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + "Query", "fff min", "matches", "rg min", "matches", "fff/rg" + ); + eprintln!( + " {:-<22}-+-{:-<9}-{:-<10}-+-{:-<9}-{:-<10}-+-{:-<7}", + "", "", "", "", "", "" + ); + + let mut fff_page_total = Duration::ZERO; + let mut rg_page_total = Duration::ZERO; + + for (name, query, ci) in &queries { + let q = *query; + let ci = *ci; + let fs = run_n(|| run_fff_page(&picker, q), iters); + let rs = run_n(|| run_rg_page(&canonical, q, ci, 50, threads), iters); + + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + name, + fmt_dur(fs.min), + fs.count, + fmt_dur(rs.min), + rs.count, + ratio_str(fs.min, rs.min), + ); + + fff_page_total += fs.min; + rg_page_total += rs.min; + } + + eprintln!( + " {:<22} | {:>9} {:>10} | {:>9} {:>10} | {:>7}", + "TOTAL", + fmt_dur(fff_page_total), + "", + fmt_dur(rg_page_total), + "", + ratio_str(fff_page_total, rg_page_total), + ); + + eprintln!( + "\n=== Summary (total min across all queries, {} iterations) ===\n", + iters + ); + eprintln!( + " {:>25} | {:>12} | {:>12} | {:>7}", + "", "fff", "rg", "speedup" + ); + eprintln!(" {:->25}-+-{:->12}-+-{:->12}-+-{:->7}", "", "", "", ""); + eprintln!( + " {:>25} | {:>12} | {:>12} | {:>7}", + "full results (collect)", + fmt_dur(fff_full_total), + fmt_dur(rg_full_total), + ratio_str(fff_full_total, rg_full_total), + ); + eprintln!( + " {:>25} | {:>12} | {:>12} | {:>7}", + "first-page (UI latency)", + fmt_dur(fff_page_total), + fmt_dur(rg_page_total), + ratio_str(fff_page_total, rg_page_total), + ); + + eprintln!(); + eprintln!(" Note: rg cost includes fork/exec + directory traversal + gitignore parsing"); + eprintln!(" on EVERY invocation (= every keystroke in telescope/fzf-lua)."); + eprintln!(" fff pays this cost once at startup, then searches from warm cached mmaps."); + eprintln!(); +} diff --git a/crates/fff-nvim/src/bin/jemalloc_profile.rs b/crates/fff-nvim/src/bin/jemalloc_profile.rs new file mode 100644 index 0000000..8ec63bf --- /dev/null +++ b/crates/fff-nvim/src/bin/jemalloc_profile.rs @@ -0,0 +1,289 @@ +use fff::file_picker::{FFFMode, FilePicker}; +use fff::{FuzzySearchOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency}; +use std::env; +use std::thread; +use std::time::Duration; + +fn get_mem_stat() -> Result<(usize, usize, usize), Box> { + // Use system memory info since jemalloc-ctl conflicts + #[cfg(target_os = "macos")] + { + use std::process::Command; + let pid = std::process::id(); + let output = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output()?; + + let rss_str = String::from_utf8(output.stdout)?; + let rss_kb: usize = rss_str.trim().parse()?; + let rss_bytes = rss_kb * 1024; + Ok((rss_bytes, rss_bytes, rss_bytes)) + } + + #[cfg(target_os = "linux")] + { + let pid = std::process::id(); + let status_path = format!("/proc/{}/status", pid); + let content = std::fs::read_to_string(status_path)?; + + for line in content.lines() { + if line.starts_with("VmRSS:") { + let parts: Vec<&str> = line.split_whitespace().collect(); + if let Ok(rss_kb) = parts[1].parse::() + && parts.len() >= 2 + { + let rss_bytes = rss_kb * 1024; + return Ok((rss_bytes, rss_bytes, rss_bytes)); + } + } + } + Err("Could not find VmRSS in /proc/pid/status".into()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + Ok((0, 0, 0)) + } +} + +fn format_bytes(bytes: usize) -> String { + const UNITS: &[&str] = &["B", "KB", "MB", "GB"]; + let mut size = bytes as f64; + let mut unit_index = 0; + + while size >= 1024.0 && unit_index < UNITS.len() - 1 { + size /= 1024.0; + unit_index += 1; + } + + format!("{:.2} {}", size, UNITS[unit_index]) +} + +fn test_search_memory_pattern( + shared_picker: &SharedFilePicker, + name: &str, + iterations: usize, + query_pattern: impl Fn(usize) -> String, +) -> Result<(), Box> { + println!("🧪 {}", name); + + let (initial_allocated, initial_active, initial_resident) = get_mem_stat()?; + println!( + " Initial - Allocated: {}, Active: {}, Resident: {}", + format_bytes(initial_allocated), + format_bytes(initial_active), + format_bytes(initial_resident) + ); + + let mut max_allocated = initial_allocated; + let mut max_active = initial_active; + let mut max_resident = initial_resident; + + for i in 0..iterations { + let query = query_pattern(i); + + let (result_count, _total_matched) = { + let guard = shared_picker.read().unwrap(); + if let Some(ref picker) = *guard { + let parser = QueryParser::default(); + let parsed = parser.parse(&query); + let search_result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 1 + (i % 4), + current_file: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 50 + (i % 50), + }, + }, + ); + (search_result.items.len(), search_result.total_matched) + } else { + continue; + } + }; + + if i % 100 == 0 { + let (allocated, active, resident) = get_mem_stat()?; + max_allocated = max_allocated.max(allocated); + max_active = max_active.max(active); + max_resident = max_resident.max(resident); + + if i % 500 == 0 { + println!( + " Iteration {}: {} results, Allocated: {} (+{})", + i, + result_count, + format_bytes(allocated), + format_bytes(allocated.saturating_sub(initial_allocated)) + ); + } + } + + if i % 200 == 199 { + thread::sleep(Duration::from_millis(1)); + } + } + + let (final_allocated, final_active, final_resident) = get_mem_stat()?; + + println!( + " Final - Allocated: {} (+{}), Active: {} (+{}), Resident: {} (+{})", + format_bytes(final_allocated), + format_bytes(final_allocated.saturating_sub(initial_allocated)), + format_bytes(final_active), + format_bytes(final_active.saturating_sub(initial_active)), + format_bytes(final_resident), + format_bytes(final_resident.saturating_sub(initial_resident)) + ); + + println!( + " Peak - Allocated: {} (+{}), Active: {} (+{}), Resident: {} (+{})", + format_bytes(max_allocated), + format_bytes(max_allocated.saturating_sub(initial_allocated)), + format_bytes(max_active), + format_bytes(max_active.saturating_sub(initial_active)), + format_bytes(max_resident), + format_bytes(max_resident.saturating_sub(initial_resident)) + ); + + let growth_per_search = + (final_allocated.saturating_sub(initial_allocated)) as f64 / iterations as f64; + println!( + " Average growth per search: {:.2} bytes", + growth_per_search + ); + println!(); + + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + let base_path = if args.len() > 1 { + args[1].clone() + } else { + env::current_dir()?.to_str().unwrap_or(".").to_string() + }; + + println!("🧪 FFF.nvim Jemalloc Memory Profiler"); + println!("===================================="); + println!("Test directory: {}", base_path); + println!(); + + // Create shared state + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + // Initialize FilePicker + println!("Initializing FilePicker..."); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + fff::FilePickerOptions { + base_path: base_path.clone(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + ..Default::default() + }, + )?; + + // Wait for initial scan + println!("Waiting for file scan..."); + loop { + if let Ok(guard) = shared_picker.read() + && let Some(ref picker) = *guard + && !picker.is_scan_active() + && !picker.get_files().is_empty() + { + break; + } + thread::sleep(Duration::from_millis(100)); + } + + let file_count = { + let guard = shared_picker.read().unwrap(); + guard.as_ref().unwrap().get_files().len() + }; + + println!("📊 Found {} files", file_count); + + let (initial_allocated, initial_active, initial_resident) = get_mem_stat()?; + println!("🧠 Baseline memory:"); + println!(" Allocated: {}", format_bytes(initial_allocated)); + println!(" Active: {}", format_bytes(initial_active)); + println!(" Resident: {}", format_bytes(initial_resident)); + println!(); + + // Test different memory patterns + + // 1. Repeated same query - should have minimal growth if caching works + test_search_memory_pattern(&shared_picker, "Same Query Repeated (1000x)", 1000, |_| { + "test".to_string() + })?; + + // 2. Cycling through different queries + test_search_memory_pattern(&shared_picker, "Cycling Queries (1000x)", 1000, |i| { + let queries = [ + "test", "main", "lib", "src", "mod", "file", "picker", "fuzzy", "search", + ]; + queries[i % queries.len()].to_string() + })?; + + // 3. Unique queries each time - worst case for any caching + test_search_memory_pattern(&shared_picker, "Unique Queries (500x)", 500, |i| { + format!("unique_query_{}", i) + })?; + + // 4. Queries that return many results + test_search_memory_pattern( + &shared_picker, + "High Result Count (500x)", + 500, + |_| "a".to_string(), // Single character likely to match many files + )?; + + // 5. Queries with no results + test_search_memory_pattern(&shared_picker, "No Results (500x)", 500, |_| { + "zzzz_no_match_expected".to_string() + })?; + + // 6. Long intensive test + test_search_memory_pattern(&shared_picker, "Long Intensive Test (2000x)", 2000, |i| { + let patterns = [ + "rs", "lua", "toml", "mod", "lib", "main", "test", "src", "file", + ]; + format!("{}{}", patterns[i % patterns.len()], i % 100) + })?; + + let (final_allocated, final_active, final_resident) = get_mem_stat()?; + + println!("🏁 Final Memory Stats:"); + println!( + " Allocated: {} (growth: {})", + format_bytes(final_allocated), + format_bytes(final_allocated.saturating_sub(initial_allocated)) + ); + println!( + " Active: {} (growth: {})", + format_bytes(final_active), + format_bytes(final_active.saturating_sub(initial_active)) + ); + println!( + " Resident: {} (growth: {})", + format_bytes(final_resident), + format_bytes(final_resident.saturating_sub(initial_resident)) + ); + + let total_searches = 5500; + let avg_growth = + (final_allocated.saturating_sub(initial_allocated)) as f64 / total_searches as f64; + println!(" Average growth per search: {:.2} bytes", avg_growth); + + Ok(()) +} diff --git a/crates/fff-nvim/src/bin/search_profiler.rs b/crates/fff-nvim/src/bin/search_profiler.rs new file mode 100644 index 0000000..869393a --- /dev/null +++ b/crates/fff-nvim/src/bin/search_profiler.rs @@ -0,0 +1,140 @@ +use fff::file_picker::{FFFMode, FilePicker}; +use fff::{FuzzySearchOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency}; +use std::time::{Duration, Instant}; + +/// Wait for background scan to complete +fn wait_for_scan(shared_picker: &SharedFilePicker, timeout_secs: u64) -> Result { + let timeout = Duration::from_secs(timeout_secs); + if !shared_picker.wait_for_scan(timeout) { + return Err(format!("Scan timed out after {} seconds", timeout_secs)); + } + + let picker_guard = shared_picker + .read() + .map_err(|e| format!("Failed to acquire read lock: {}", e))?; + if let Some(ref picker) = *picker_guard { + Ok(picker.get_files().len()) + } else { + Err("FilePicker not initialized".to_string()) + } +} + +fn main() { + let big_repo_path = std::path::PathBuf::from("./big-repo"); + + if !big_repo_path.exists() { + eprintln!( + "./big-repo directory does not exist. Run git clone https://github.com/torvalds/linux.git big-repo" + ); + return; + } + + let canonical_path = + fff::path_utils::canonicalize(&big_repo_path).expect("Failed to canonicalize path"); + + // Create shared state + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + eprintln!("Initializing FilePicker for: {:?}", canonical_path); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + fff::FilePickerOptions { + base_path: canonical_path.to_string_lossy().to_string(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + ..Default::default() + }, + ) + .expect("Failed to init FilePicker with shared state"); + + // Give background thread time to start + std::thread::sleep(Duration::from_millis(200)); + + eprintln!("Waiting for scan to complete..."); + let file_count = wait_for_scan(&shared_picker, 120).expect("Failed to wait for scan"); + eprintln!("✓ Indexed {} files\n", file_count); + + let picker_guard = shared_picker.read().expect("Failed to acquire read lock"); + let picker = picker_guard.as_ref().expect("FilePicker not initialized"); + + // Test queries representing different search patterns + let test_queries = vec![ + ("short_common", "mod", 100), + ("medium_specific", "controller", 100), + ("long_rare", "user_authentication", 100), + ("typo_resistant", "contrlr", 100), + ("path_like", "src/lib", 100), + ("single_char", "a", 100), + ("two_char", "st", 100), + ("partial_word", "test", 100), + ("deep_path", "drivers/net", 100), + ("extension", ".rs", 100), + ]; + + eprintln!("Running search profiler..."); + eprintln!("Query | Iterations | Total Time | Avg Time | Matches"); + eprintln!("----------------------|------------|------------|-----------|--------"); + + let global_start = Instant::now(); + let mut total_iterations = 0; + + for (name, query, iterations) in test_queries { + let start = Instant::now(); + let mut match_count = 0; + let parser = QueryParser::default(); + + for _ in 0..iterations { + let parsed = parser.parse(query); + let results = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 4, + current_file: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 100, + }, + }, + ); + + match_count += results.total_matched; + } + + let elapsed = start.elapsed(); + let avg_time = elapsed / iterations as u32; + + eprintln!( + "{:<21} | {:>10} | {:>9.2}s | {:>7}µs | {}", + name, + iterations, + elapsed.as_secs_f64(), + avg_time.as_micros(), + match_count / iterations + ); + + total_iterations += iterations; + } + + let total_time = global_start.elapsed(); + + eprintln!("\n=== Summary ==="); + eprintln!("Total searches: {}", total_iterations); + eprintln!("Total time: {:.2}s", total_time.as_secs_f64()); + eprintln!( + "Average per search: {}µs", + (total_time.as_micros() as usize) / total_iterations + ); + eprintln!( + "Searches per sec: {:.0}", + total_iterations as f64 / total_time.as_secs_f64() + ); + + // Keep the program alive briefly so perf can capture everything + std::thread::sleep(Duration::from_millis(100)); +} diff --git a/crates/fff-nvim/src/bin/test_memory_leak.rs b/crates/fff-nvim/src/bin/test_memory_leak.rs new file mode 100644 index 0000000..67e1c47 --- /dev/null +++ b/crates/fff-nvim/src/bin/test_memory_leak.rs @@ -0,0 +1,282 @@ +use fff::file_picker::{FFFMode, FilePicker}; +use fff::{FuzzySearchOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency}; +use std::env; +use std::io::{self, Write}; +use std::thread; +use std::time::{Duration, Instant}; + +fn get_memory_usage() -> Result> { + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + return Err("Memory usage check is only supported on Linux and macOS".into()); + } + + #[cfg(target_os = "macos")] + { + let pid = std::process::id(); + use std::process::Command; + let output = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output()?; + + let rss_str = String::from_utf8(output.stdout)?; + let rss_kb: u64 = rss_str.trim().parse()?; + + Ok(rss_kb * 1024) + } + + #[cfg(target_os = "linux")] + { + let pid = std::process::id(); + let status_path = format!("/proc/{}/status", pid); + let content = std::fs::read_to_string(status_path)?; + + for line in content.lines() { + if line.starts_with("VmRSS:") { + let parts: Vec<&str> = line.split_whitespace().collect(); + if let Ok(rss_kb) = parts[1].parse::() + && parts.len() >= 2 + { + return Ok(rss_kb * 1024); // Convert KB to bytes + } + } + } + + Err("Could not find VmRSS in /proc/pid/status".into()) + } +} + +fn format_bytes(bytes: u64) -> String { + const UNITS: &[&str] = &["B", "KB", "MB", "GB"]; + let mut size = bytes as f64; + let mut unit_index = 0; + + while size >= 1024.0 && unit_index < UNITS.len() - 1 { + size /= 1024.0; + unit_index += 1; + } + + format!("{:.2} {}", size, UNITS[unit_index]) +} + +fn main() -> Result<(), Box> { + if !cfg!(target_os = "linux") && !cfg!(target_os = "macos") { + eprintln!("This test is only supported on Linux and macOS."); + std::process::exit(1); + } + + let args: Vec = env::args().collect(); + let base_path = if args.len() > 1 { + args[1].clone() + } else { + env::current_dir()?.to_str().unwrap_or(".").to_string() + }; + + println!("🧪 FFF.nvim Memory Leak Test (Using Crate)"); + println!("=========================================="); + println!("Test directory: {}", base_path); + println!(); + + // Create shared state + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + // Initialize the file picker + println!("📁 Initializing FilePicker..."); + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + fff::FilePickerOptions { + base_path: base_path.clone(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + ..Default::default() + }, + )?; + + // Wait for initial scan to complete + println!("⏳ Waiting for initial file scan to complete..."); + let mut wait_time = 0; + let mut scan_completed = false; + + loop { + if let Ok(guard) = shared_picker.read() + && let Some(ref picker) = *guard + { + if !picker.is_scan_active() { + println!("Scan inactive, checking file count..."); + let file_count = picker.get_files().len(); + if file_count > 0 { + println!("Async scan found {} files", file_count); + scan_completed = true; + break; + } + } else { + println!("Scan active, waiting..."); + } + } + thread::sleep(Duration::from_millis(100)); + wait_time += 100; + if wait_time > 10000 { + println!("Timeout waiting for async scan, triggering manual scan..."); + break; + } + } + + // If async scan didn't work, trigger a manual scan + if !scan_completed { + println!("Triggering manual rescan..."); + let _ = shared_picker.trigger_full_rescan_async(&shared_frecency); + } + + let initial_file_count = { + let guard = shared_picker.read().unwrap(); + if let Some(ref picker) = *guard { + let files = picker.get_files(); + println!("Found {} files in picker", files.len()); + if !files.is_empty() { + println!("Sample files:"); + for (i, file) in files.iter().take(5).enumerate() { + println!(" {}. {}", i + 1, file.relative_path(picker)); + } + } + files.len() + } else { + println!("No picker found!"); + 0 + } + }; + + println!( + "📊 Initial scan complete: {} files found", + initial_file_count + ); + + if initial_file_count == 0 { + eprintln!("❌ No files found in directory. Cannot proceed with memory test."); + std::process::exit(1); + } + + // Record initial memory usage + let initial_memory = get_memory_usage().unwrap_or(0); + println!("🧠 Initial memory usage: {}", format_bytes(initial_memory)); + println!(); + + // Test queries to cycle through + let test_queries = vec![ + "rs", "mod", "lib", "main", "test", "src", "lua", "cargo", "toml", "init", "config", + "util", "file", "picker", "fuzzy", "search", "git", "fn", "struct", "impl", "pub", "use", + "let", "match", "if", "for", "async", "await", "Result", "Error", "Vec", "String", + "Option", + ]; + + let mut last_memory_check = Instant::now(); + let mut peak_memory = initial_memory; + let mut memory_samples = Vec::new(); + + println!("🔥 Starting intensive search test..."); + println!("Press Ctrl+C to stop the test"); + println!(); + let mut search_count = 0; + + while search_count < 1000 { + search_count += 1; + + // Perform search directly on FilePicker + let query = test_queries[search_count % test_queries.len()]; + let max_results = 50 + (search_count % 100); // Vary result count + let max_threads = 1 + (search_count % 8); // Vary thread count + + let search_start = Instant::now(); + let parser = QueryParser::default(); + let (result_count, search_duration) = { + let guard = shared_picker.read().unwrap(); + if let Some(ref picker) = *guard { + let parsed = parser.parse(query); + let search_result = picker.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads, + current_file: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: max_results, + }, + }, + ); + let duration = search_start.elapsed(); + (search_result.items.len(), duration) + } else { + (0, search_start.elapsed()) + } + }; + + search_count += 1; + + // Check memory every 100 searches or every 5 seconds + let now = Instant::now(); + if (search_count % 100 == 0 + || now.duration_since(last_memory_check) > Duration::from_secs(5)) + && let Ok(current_memory) = get_memory_usage() + { + memory_samples.push(current_memory); + + if current_memory > peak_memory { + peak_memory = current_memory; + } + + let memory_growth = current_memory.saturating_sub(initial_memory); + + println!( + "🔍 Search #{}: '{}' -> {} results in {:?} | Memory: {} (+{}) | Peak: {}", + search_count, + query, + result_count, + search_duration, + format_bytes(current_memory), + format_bytes(memory_growth), + format_bytes(peak_memory) + ); + + last_memory_check = now; + + // Calculate memory growth trend over last 10 samples + if memory_samples.len() >= 10 { + let recent_samples = &memory_samples[memory_samples.len() - 10..]; + let first_recent = recent_samples[0]; + let last_recent = recent_samples[recent_samples.len() - 1]; + + if last_recent > first_recent { + let recent_growth = last_recent - first_recent; + if recent_growth > 1024 * 1024 { + // More than 1MB growth in recent samples + println!( + "⚠️ POTENTIAL LEAK: Recent memory growth: {}", + format_bytes(recent_growth) + ); + } + } + } + + io::stdout().flush().unwrap(); + } + + // Brief pause to prevent overwhelming the system + if search_count % 10 == 0 { + thread::sleep(Duration::from_millis(1)); + } + + // Force cleanup test every 1000 searches + if search_count % 1000 == 0 { + println!("🧹 Forcing potential cleanup at search #{}", search_count); + // Force garbage collection by creating and dropping temporary data + thread::sleep(Duration::from_millis(10)); + } + } + + Ok(()) +} diff --git a/crates/fff-nvim/src/bin/test_watcher.rs b/crates/fff-nvim/src/bin/test_watcher.rs new file mode 100644 index 0000000..be717b0 --- /dev/null +++ b/crates/fff-nvim/src/bin/test_watcher.rs @@ -0,0 +1,205 @@ +#![allow(clippy::all)] +#![allow(dead_code)] +#![allow(clippy::enum_variant_names)] + +use fff::file_picker::FilePicker; +use fff::git::format_git_status; +use fff::{ + FFFMode, FuzzySearchOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency, +}; +use std::env; +use std::io::{self, Write}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::Duration; + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + let base_path = if args.len() > 1 { + args[1].clone() + } else { + env::current_dir()?.to_str().unwrap_or(".").to_string() + }; + + // Set up signal handler for graceful shutdown + let running = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + + // Create shared state + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + + // Clone for signal handler + let picker_for_cleanup = shared_picker.clone(); + ctrlc::set_handler(move || { + println!("\n🛑 Received interrupt signal, shutting down..."); + if let Ok(mut guard) = picker_for_cleanup.write() { + guard.take(); + } + println!("🧹 FilePicker cleaned up"); + r.store(false, Ordering::SeqCst); + std::process::exit(0); + })?; + + let mut git_stats = std::collections::HashMap::new(); + + // Initialize the file picker using shared state + FilePicker::new_with_shared_state( + shared_picker.clone(), + shared_frecency.clone(), + fff::FilePickerOptions { + base_path: base_path.clone(), + enable_mmap_cache: false, + mode: FFFMode::default(), + ..Default::default() + }, + )?; + + // Get initial file count from shared state + let initial_count = { + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let files = picker.get_files(); + println!("Initial file count: {}", files.len()); + + if !files.is_empty() { + println!("Sample files:"); + for (i, file) in files.iter().take(5).enumerate() { + println!( + " {}. {} ({})", + i + 1, + file.relative_path(picker), + format_git_status(file.git_status) + ); + } + if files.len() > 5 { + println!(" ... and {} more files", files.len() - 5); + } + } + files.len() + }; + + println!("{:=<60}", ""); + println!("🔴 LIVE FILE MONITORING - Press Ctrl+C to stop"); + println!("{:=<60}", ""); + + let mut last_count = initial_count; + let mut iteration = 0; + + while running.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(500)); + iteration += 1; + + let current_count = { + let guard = shared_picker.read().unwrap(); + guard.as_ref().unwrap().get_files().len() + }; + + if current_count != last_count { + let timestamp = chrono::Local::now().format("%H:%M:%S%.3f"); + + if current_count > last_count { + let added = current_count - last_count; + println!( + "🟢 [{}] +{} files added | Total: {}", + timestamp, added, current_count + ); + + // Show some recently added files + let guard = shared_picker.read().unwrap(); + let picker = guard.as_ref().unwrap(); + let files = picker.get_files(); + let newest_files = files.iter().rev().take(added.min(3)); + for file in newest_files { + println!(" ➕ {}", file.relative_path(picker)); + } + } else { + let removed = last_count - current_count; + println!( + "🔴 [{}] -{} files removed | Total: {}", + timestamp, removed, current_count + ); + } + + last_count = current_count; + } + + if iteration % 20 == 0 { + let timestamp = chrono::Local::now().format("%H:%M:%S"); + println!( + "💓 [{}] Heartbeat - {} files cached, watcher active", + timestamp, current_count + ); + + let guard = shared_picker.read().unwrap(); + let current_files = guard.as_ref().unwrap().get_files(); + + git_stats.clear(); + for file in current_files { + let status = format_git_status(file.git_status); + *git_stats.entry(status).or_insert(0) += 1; + } + + if !git_stats.is_empty() { + print!(" 📊 Git status: "); + for (status, count) in &git_stats { + print!("{}:{} ", status, count); + } + println!(); + } + } + + if iteration % 40 == 0 { + let timestamp = chrono::Local::now().format("%H:%M:%S"); + let guard = shared_picker.read().unwrap(); + let picker_ref = guard.as_ref().unwrap(); + let parser = QueryParser::default(); + let parsed = parser.parse("rs"); + let search_results = picker_ref.fuzzy_search( + &parsed, + None, + FuzzySearchOptions { + max_threads: 2, + current_file: None, + project_path: None, + combo_boost_score_multiplier: 100, + min_combo_count: 3, + pagination: PaginationArgs { + offset: 0, + limit: 5, + }, + }, + ); + + println!( + "🔍 [{}] Search test 'rs': {} matches", + timestamp, + search_results.items.len() + ); + for (i, (file, score)) in search_results + .items + .iter() + .zip(search_results.scores.iter()) + .take(3) + .enumerate() + { + println!( + " {}. {} (score: {})", + i + 1, + file.relative_path(picker_ref), + score.total + ); + } + } + + io::stdout().flush().unwrap(); + } + + // Clean up before exit + if let Ok(mut guard) = shared_picker.write() { + guard.take(); + } + + Ok(()) +} diff --git a/crates/fff-nvim/src/error.rs b/crates/fff-nvim/src/error.rs new file mode 100644 index 0000000..91a09e7 --- /dev/null +++ b/crates/fff-nvim/src/error.rs @@ -0,0 +1,26 @@ +//! Error handling for fff-nvim +//! +//! This module provides utilities for converting fff errors to mlua errors. + +use fff::Error as CoreError; + +/// Convert a fff::Error to mlua::Error +/// +/// This function is used because we can't implement From for mlua::Error +/// due to Rust's orphan rules (both types are foreign to this crate). +pub fn to_lua_error(err: CoreError) -> mlua::Error { + let string_value = err.to_string(); + ::tracing::error!(string_value); + mlua::Error::RuntimeError(string_value) +} + +/// Extension trait for Result to convert to LuaResult +pub trait IntoLuaResult { + fn into_lua_result(self) -> mlua::Result; +} + +impl IntoLuaResult for Result { + fn into_lua_result(self) -> mlua::Result { + self.map_err(to_lua_error) + } +} diff --git a/crates/fff-nvim/src/hex_dump.rs b/crates/fff-nvim/src/hex_dump.rs new file mode 100644 index 0000000..05c1193 --- /dev/null +++ b/crates/fff-nvim/src/hex_dump.rs @@ -0,0 +1,190 @@ +use mlua::prelude::*; +use std::fmt::Write as _; +use std::io::{Read, Seek, SeekFrom}; + +// Byte category colors (matching hexyl's default theme) +const COLOR_OFFSET: &str = "#888888"; +const COLOR_NULL: &str = "#555753"; +const COLOR_ASCII_PRINTABLE: &str = "#06989a"; +const COLOR_ASCII_WHITESPACE: &str = "#4e9a06"; +const COLOR_ASCII_OTHER: &str = "#4e9a06"; +const COLOR_NON_ASCII: &str = "#c4a000"; + +fn byte_color(b: u8) -> &'static str { + match b { + 0x00 => COLOR_NULL, + 0x20 | 0x09 | 0x0a | 0x0d => COLOR_ASCII_WHITESPACE, + 0x21..=0x7e => COLOR_ASCII_PRINTABLE, + 0x01..=0x1f | 0x7f => COLOR_ASCII_OTHER, + _ => COLOR_NON_ASCII, + } +} + +fn byte_char(b: u8) -> char { + match b { + 0x20..=0x7e => b as char, + _ => '.', + } +} + +const BYTES_PER_LINE: usize = 16; + +struct Span { + line: usize, + col_start: usize, + col_end: usize, + color: &'static str, +} + +/// Push a span, merging with the previous one if same line and color. +fn push_span( + spans: &mut Vec, + line: usize, + col_start: usize, + col_end: usize, + color: &'static str, +) { + if let Some(last) = spans.last_mut() { + // Merge if same line, same color, and adjacent (allow small gaps for spaces between hex pairs) + if last.line == line && std::ptr::eq(last.color, color) && col_start <= last.col_end + 1 { + last.col_end = col_end; + return; + } + } + spans.push(Span { + line, + col_start, + col_end, + color, + }); +} + +/// Format raw bytes into hex dump lines with coalesced highlight spans. +/// +/// Layout per line: +/// ```text +/// XXXXXXXX HH HH HH HH HH HH HH HH HH HH HH HH HH HH HH HH CCCCCCCCCCCCCCCC +/// ``` +fn format_hex_dump(raw_bytes: &[u8], base_offset: u64) -> (Vec, Vec) { + let mut lines = Vec::new(); + let mut spans = Vec::new(); + + for (chunk_idx, chunk) in raw_bytes.chunks(BYTES_PER_LINE).enumerate() { + let addr = base_offset + (chunk_idx * BYTES_PER_LINE) as u64; + let mut line = format!("{addr:08x} "); + + // Offset label highlight + push_span(&mut spans, chunk_idx, 0, 8, COLOR_OFFSET); + + // Hex pairs with a gap after 8 bytes + for (i, &b) in chunk.iter().enumerate() { + if i == 8 { + line.push(' '); + } + let col = line.len(); + push_span(&mut spans, chunk_idx, col, col + 2, byte_color(b)); + write!(line, "{b:02x} ").unwrap(); + } + + // Pad if the last line is short + if chunk.len() < BYTES_PER_LINE { + let missing = BYTES_PER_LINE - chunk.len(); + let mut pad = missing * 3; + if chunk.len() <= 8 { + pad += 1; + } + for _ in 0..pad { + line.push(' '); + } + } + + // Separator before char panel + line.push(' '); + + // Character panel — consecutive same-color chars merge automatically + let char_start = line.len(); + for (i, &b) in chunk.iter().enumerate() { + let col = char_start + i; + push_span(&mut spans, chunk_idx, col, col + 1, byte_color(b)); + line.push(byte_char(b)); + } + + lines.push(line); + } + + (lines, spans) +} + +/// Generate a hex dump for a binary file with paging support and highlight data. +/// +/// Returns a Lua table: +/// ```text +/// { +/// lines: string[], +/// highlights: {line_0idx, col_start, col_end, color}[], +/// has_more: bool, +/// next_offset: number, +/// } +/// ``` +pub fn hex_dump( + lua: &Lua, + (file_path, offset, length): (String, Option, Option), +) -> LuaResult { + let offset = offset.unwrap_or(0); + let length = length.unwrap_or(4096); + + let file = std::fs::File::open(&file_path) + .map_err(|e| LuaError::RuntimeError(format!("Failed to open file: {e}")))?; + + let file_size = file + .metadata() + .map_err(|e| LuaError::RuntimeError(format!("Failed to get metadata: {e}")))? + .len(); + + let table = lua.create_table()?; + + if offset >= file_size { + table.set("lines", lua.create_table()?)?; + table.set("highlights", lua.create_table()?)?; + table.set("has_more", false)?; + table.set("next_offset", file_size)?; + return Ok(LuaValue::Table(table)); + } + + let mut reader = std::io::BufReader::new(file); + reader + .seek(SeekFrom::Start(offset)) + .map_err(|e| LuaError::RuntimeError(format!("Failed to seek: {e}")))?; + let mut raw_bytes = Vec::with_capacity(length as usize); + reader + .by_ref() + .take(length) + .read_to_end(&mut raw_bytes) + .map_err(|e| LuaError::RuntimeError(format!("Failed to read: {e}")))?; + + let (plain_lines, hl_spans) = format_hex_dump(&raw_bytes, offset); + + let lines_table = lua.create_table()?; + for (i, line) in plain_lines.iter().enumerate() { + lines_table.set(i + 1, line.as_str())?; + } + table.set("lines", lines_table)?; + + let highlights_table = lua.create_table()?; + for (i, span) in hl_spans.iter().enumerate() { + let hl = lua.create_table()?; + hl.raw_set(1, span.line)?; + hl.raw_set(2, span.col_start)?; + hl.raw_set(3, span.col_end)?; + hl.raw_set(4, span.color)?; + highlights_table.raw_set(i + 1, hl)?; + } + table.set("highlights", highlights_table)?; + + let bytes_read = raw_bytes.len() as u64; + let next_offset = offset + bytes_read; + table.set("has_more", next_offset < file_size)?; + table.set("next_offset", next_offset)?; + + Ok(LuaValue::Table(table)) +} diff --git a/crates/fff-nvim/src/lib.rs b/crates/fff-nvim/src/lib.rs new file mode 100644 index 0000000..a9f3cb2 --- /dev/null +++ b/crates/fff-nvim/src/lib.rs @@ -0,0 +1,1034 @@ +use crate::path_shortening::shorten_path_with_cache; +use error::IntoLuaResult; +use fff::file_picker::FilePicker; +use fff::frecency::FrecencyTracker; +use fff::path_utils::expand_tilde; +use fff::query_tracker::QueryTracker; +use fff::{ + DbHealthChecker, DirSearchConfig, Error, FFFMode, FFFQuery, FileSearchConfig, + FuzzySearchOptions, MixedSearchConfig, PaginationArgs, QueryParser, Score, SearchResult, + SharedFilePicker, SharedFrecency, SharedQueryTracker, +}; +use mimalloc::MiMalloc; +use mlua::prelude::*; +use once_cell::sync::Lazy; +use path_shortening::PathShortenStrategy; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use user_config::{NvimGrepConfig, UserConfigOptions, set_global_user_config}; + +mod error; +mod hex_dump; +mod log; +mod lua_types; +mod path_shortening; +mod user_config; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +// the global state for neovim lives here for efficiency +// lua ffi is pretty bad with the overhead of converting raw pointer into tables +pub static FILE_PICKER: Lazy = Lazy::new(SharedFilePicker::default); +pub static FRECENCY: Lazy = Lazy::new(SharedFrecency::default); +pub static QUERY_TRACKER: Lazy = Lazy::new(SharedQueryTracker::default); + +pub fn init_db( + _: &Lua, + (frecency_db_path, history_db_path, _use_unsafe_no_lock): (String, String, bool), +) -> LuaResult { + // Route through SharedFrecency::init / SharedQueryTracker::init so the + // GC thread gets spawned + bound to the shared handle's RwLock. + FRECENCY + .init(FrecencyTracker::open(&frecency_db_path).into_lua_result()?) + .into_lua_result()?; + tracing::info!("Frecency database initialized at {}", frecency_db_path); + + QUERY_TRACKER + .init(QueryTracker::open(&history_db_path).into_lua_result()?) + .into_lua_result()?; + tracing::info!("Query tracker database initialized at {}", history_db_path); + Ok(true) +} + +pub fn destroy_frecency_db(_: &Lua, _: ()) -> LuaResult { + Ok(FRECENCY.destroy().into_lua_result()?.is_some()) +} + +pub fn destroy_query_db(_: &Lua, _: ()) -> LuaResult { + Ok(QUERY_TRACKER.destroy().into_lua_result()?.is_some()) +} + +/// Opts table accepted by `init_file_picker` / `restart_index_in_path`. +/// Backwards compat: positional `follow_symlinks: bool` argument still works +/// — callers that pass a table use the named fields instead. +#[derive(Default)] +struct PickerInitOpts { + follow_symlinks: bool, + enable_fs_root_scanning: bool, + enable_home_dir_scanning: bool, + enable_filename_constraint: bool, +} + +impl PickerInitOpts { + fn from_lua_value(value: Option) -> LuaResult { + let Some(value) = value else { + return Ok(Self::default()); + }; + match value { + mlua::Value::Nil => Ok(Self::default()), + mlua::Value::Boolean(b) => Ok(Self { + follow_symlinks: b, + ..Default::default() + }), + mlua::Value::Table(t) => Ok(Self { + follow_symlinks: t.get::>("follow_symlinks")?.unwrap_or(false), + enable_fs_root_scanning: t + .get::>("enable_fs_root_scanning")? + .unwrap_or(false), + enable_home_dir_scanning: t + .get::>("enable_home_dir_scanning")? + .unwrap_or(false), + enable_filename_constraint: t + .get::>("enable_filename_constraint")? + .unwrap_or(false), + }), + other => Err(LuaError::RuntimeError(format!( + "init opts must be a table, boolean, or nil — got {}", + other.type_name() + ))), + } + } +} + +pub fn init_file_picker( + _: &Lua, + (base_path, opts): (String, Option), +) -> LuaResult { + { + let guard = FILE_PICKER.read().into_lua_result()?; + if guard.is_some() { + return Ok(false); + } + } + + let opts = PickerInitOpts::from_lua_value(opts)?; + set_global_user_config(UserConfigOptions { + enable_filename_constraint: opts.enable_filename_constraint, + }); + + FilePicker::new_with_shared_state( + FILE_PICKER.clone(), + FRECENCY.clone(), + fff::FilePickerOptions { + base_path, + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + follow_symlinks: opts.follow_symlinks, + enable_fs_root_scanning: opts.enable_fs_root_scanning, + enable_home_dir_scanning: opts.enable_home_dir_scanning, + ..Default::default() + }, + ) + .into_lua_result()?; + + Ok(true) +} + +pub fn restart_index_in_path( + _: &Lua, + (new_path, opts): (String, Option), +) -> LuaResult<()> { + let path = std::path::PathBuf::from(&new_path); + if !path.exists() { + return Err(LuaError::RuntimeError(format!( + "Path does not exist: {}", + new_path + ))); + } + + let canonical_path = fff::path_utils::canonicalize(&path).map_err(|e| { + LuaError::RuntimeError(format!("Failed to canonicalize path '{}': {}", new_path, e)) + })?; + + let opts = PickerInitOpts::from_lua_value(opts)?; + set_global_user_config(UserConfigOptions { + enable_filename_constraint: opts.enable_filename_constraint, + }); + + // Spawn a background thread BEFORE touching the picker lock. The + // same-dir short-circuit previously called `FILE_PICKER.read()` on + // the lua/UI thread, which blocks if a reindex writer is already in + // flight — on repeated DirChanged events (e.g. LSP root switching + // right after closing the picker) that could freeze the nvim main + // loop for the entire duration of an in-progress scan. Move the + // check into the spawned worker so the main thread always returns + // instantly; the internal reinit path also re-checks and no-ops if + // the picker is already pointing at `canonical_path`. + std::thread::spawn(move || { + ::tracing::info!( + ?canonical_path, + "restart_index_in_path: spawned worker running" + ); + + // Inherit current picker's scanning flags when caller didn't pass + // explicit opts — otherwise a `:cd ~` after init would silently lose + // the user's `enable_home_dir_scanning = true` setting. + let (follow_symlinks, fs_root, home_dir) = { + let guard = match FILE_PICKER.read() { + Ok(g) => g, + Err(_) => return, + }; + + if let Some(ref picker) = *guard + && picker.base_path() == canonical_path + { + tracing::info!(?canonical_path, "restart_index_in_path: same dir, skipping"); + return; + } + + match guard.as_ref() { + Some(p) => ( + p.follows_symlinks() || opts.follow_symlinks, + p.fs_root_scanning_enabled() || opts.enable_fs_root_scanning, + p.home_dir_scanning_enabled() || opts.enable_home_dir_scanning, + ), + None => ( + opts.follow_symlinks, + opts.enable_fs_root_scanning, + opts.enable_home_dir_scanning, + ), + } + }; + + ::tracing::info!( + ?canonical_path, + "restart_index_in_path: calling reinit_file_picker_internal" + ); + + // this will AUTOMATICALLY drop the old picker within a write lock inside the implementation + // that will stop all the ongoing work and drop all the workeres + if let Err(e) = FilePicker::new_with_shared_state( + FILE_PICKER.clone(), + FRECENCY.clone(), + fff::FilePickerOptions { + base_path: canonical_path.to_string_lossy().to_string(), + enable_mmap_cache: true, + enable_content_indexing: true, + mode: FFFMode::Neovim, + follow_symlinks, + enable_fs_root_scanning: fs_root, + enable_home_dir_scanning: home_dir, + ..Default::default() + }, + ) { + tracing::error!( + ?e, + ?canonical_path, + "Failed to index directory after changing" + ); + } + }); + + Ok(()) +} + +pub fn scan_files(_: &Lua, _: ()) -> LuaResult<()> { + // Async: spawns a BG thread that walks off-lock, swaps sync_data + // under a brief write, then applies git + frecency off-lock. Returns + // immediately so the lua/UI thread never waits on the walk. + FILE_PICKER + .trigger_full_rescan_async(&FRECENCY) + .into_lua_result()?; + ::tracing::info!("scan_files rescan spawned"); + Ok(()) +} + +#[allow(clippy::type_complexity)] +pub fn fuzzy_search_files( + lua: &Lua, + ( + query, + max_threads, + current_file, + combo_boost_score_multiplier, + min_combo_count, + page_index, + page_size, + ): ( + String, + usize, + Option, + i32, + Option, + Option, + Option, + ), +) -> LuaResult { + let file_picker_guard = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker_guard else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + + let base_path = picker.base_path(); + let min_combo_count = min_combo_count.unwrap_or(3); + + let query_tracker_guard = QUERY_TRACKER.read().into_lua_result()?; + + if query_tracker_guard.as_ref().is_none() { + tracing::warn!("Query tracker not initialized"); + } + + tracing::debug!( + ?base_path, + ?query, + ?min_combo_count, + ?page_index, + ?page_size, + "Fuzzy search parameters" + ); + + let parsed_query = FFFQuery::parse(&query, FileSearchConfig); + let results = picker.fuzzy_search( + &parsed_query, + query_tracker_guard.as_ref(), + FuzzySearchOptions { + max_threads, + current_file: current_file.as_deref(), + project_path: Some(picker.base_path()), + combo_boost_score_multiplier, + min_combo_count, + pagination: PaginationArgs { + offset: page_index.unwrap_or(0), + limit: page_size.unwrap_or(0), + }, + }, + ); + + if results.items.is_empty() && query.contains(std::path::MAIN_SEPARATOR) { + let pure_query = match &parsed_query.fuzzy_query { + fff_query_parser::FuzzyQuery::Text(t) => t.trim(), + _ => query.trim(), + }; + + let path = expand_tilde(pure_query); + if path.is_absolute() && path.is_file() { + if let Some(found_file) = picker.get_file_by_path(&path) { + let found = SearchResult { + items: vec![found_file], + scores: vec![Score { + exact_match: true, + match_type: "path", + ..Default::default() + }], + total_matched: 1, + total_files: results.total_files, + location: parsed_query.location, + }; + + return lua_types::SearchResultLua::new(found, picker).into_lua(lua); + } + + return build_file_path_fallback(lua, &path, results.total_files); + } + } + + lua_types::SearchResultLua::new(results, picker).into_lua(lua) +} + +#[allow(clippy::type_complexity)] +pub fn fuzzy_search_directories( + lua: &Lua, + (query, max_threads, current_file, page_index, page_size): ( + String, + usize, + Option, + Option, + Option, + ), +) -> LuaResult { + let file_picker_guard = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker_guard else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + + let parser = QueryParser::new(DirSearchConfig); + let parsed = parser.parse(&query); + + let results = picker.fuzzy_search_directories( + &parsed, + FuzzySearchOptions { + max_threads, + current_file: current_file.as_deref(), + project_path: Some(picker.base_path()), + combo_boost_score_multiplier: 0, + min_combo_count: 0, + pagination: PaginationArgs { + offset: page_index.unwrap_or(0), + limit: page_size.unwrap_or(0), + }, + }, + ); + + lua_types::DirSearchResultLua::new(results, picker).into_lua(lua) +} + +#[allow(clippy::type_complexity)] +pub fn fuzzy_search_mixed( + lua: &Lua, + ( + query, + max_threads, + current_file, + combo_boost_score_multiplier, + min_combo_count, + page_index, + page_size, + ): ( + String, + usize, + Option, + i32, + Option, + Option, + Option, + ), +) -> LuaResult { + let file_picker_guard = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker_guard else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + + let query_tracker_guard = QUERY_TRACKER.read().into_lua_result()?; + let parser = QueryParser::new(MixedSearchConfig); + let parsed = parser.parse(&query); + + let results = picker.fuzzy_search_mixed( + &parsed, + query_tracker_guard.as_ref(), + FuzzySearchOptions { + max_threads, + current_file: current_file.as_deref(), + project_path: Some(picker.base_path()), + combo_boost_score_multiplier, + min_combo_count: min_combo_count.unwrap_or(3), + pagination: PaginationArgs { + offset: page_index.unwrap_or(0), + limit: page_size.unwrap_or(0), + }, + }, + ); + + lua_types::MixedSearchResultLua::new(results, picker).into_lua(lua) +} + +#[allow(clippy::type_complexity)] +pub fn live_grep( + lua: &Lua, + ( + query, + file_offset, + page_size, + max_file_size, + max_matches_per_file, + smart_case, + grep_mode, + time_budget_ms, + trim_whitespace, + ): ( + String, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + ), +) -> LuaResult { + let file_picker_guard = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker_guard else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + + let parsed_query = FFFQuery::parse(&query, NvimGrepConfig); + let mode = match grep_mode.as_deref() { + Some("regex") => fff::GrepMode::Regex, + Some("fuzzy") => fff::GrepMode::Fuzzy, + _ => fff::GrepMode::PlainText, // "plain" or nil or unknown + }; + + let options = fff::GrepSearchOptions { + max_file_size: max_file_size.unwrap_or(10 * 1024 * 1024), + max_matches_per_file: max_matches_per_file.unwrap_or(200), + smart_case: smart_case.unwrap_or(true), + file_offset: file_offset.unwrap_or(0), + page_limit: page_size.unwrap_or(50), + mode, + time_budget_ms: time_budget_ms.unwrap_or(0), + before_context: 0, + after_context: 0, + classify_definitions: false, + trim_whitespace: trim_whitespace.unwrap_or(false), + abort_signal: None, + }; + + let result = picker.grep(&parsed_query, &options); + lua_types::GrepResultLua::new(result, picker).into_lua(lua) +} + +/// Build a file-picker result for an absolute path that exists on disk but +/// isn't in the picker index (e.g. file from a different project). +fn build_file_path_fallback(lua: &Lua, path: &Path, total_files: usize) -> LuaResult { + let table = lua.create_table()?; + + let name = path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + let path_str = path.to_string_lossy().to_string(); + + let item = lua.create_table()?; + item.set("relative_path", path_str.as_str())?; + item.set("name", name.as_str())?; + item.set("size", path.metadata().map(|m| m.len()).unwrap_or(0))?; + item.set("modified", 0u64)?; + item.set("access_frecency_score", 0i32)?; + item.set("modification_frecency_score", 0i32)?; + item.set("total_frecency_score", 0i32)?; + item.set("git_status", "")?; + item.set("is_binary", false)?; + + let items_table = lua.create_table()?; + items_table.set(1, item)?; + table.set("items", items_table)?; + + let score = lua.create_table()?; + score.set("total", 0)?; + score.set("base_score", 0)?; + score.set("filename_bonus", 0)?; + score.set("special_filename_bonus", 0)?; + score.set("frecency_boost", 0)?; + score.set("git_status_boost", 0)?; + score.set("distance_penalty", 0)?; + score.set("current_file_penalty", 0)?; + score.set("combo_match_boost", 0)?; + score.set("exact_match", true)?; + score.set("match_type", "path")?; + + let scores_table = lua.create_table()?; + scores_table.set(1, score)?; + table.set("scores", scores_table)?; + + table.set("total_matched", 1)?; + table.set("total_files", total_files)?; + + Ok(LuaValue::Table(table)) +} + +pub fn track_access(_: &Lua, file_path: String) -> LuaResult { + // must be async and capture a local copy of the path because the write lock + // is unsafe and can be held forever which is unavoidable, but at least we can + // prevent users from deadlock of the main thread if someone deletes a lock + let file_path = PathBuf::from(&file_path); + std::thread::spawn(move || { + { + let frecency_guard = match FRECENCY.read() { + Ok(g) => g, + Err(e) => { + ::tracing::debug!(?e, "track_access: frecency read lock poisoned"); + return; + } + }; + let Some(ref frecency) = *frecency_guard else { + return; + }; + if let Err(e) = frecency.track_access(file_path.as_path()) { + ::tracing::debug!(?e, ?file_path, "track_access: frecency DB write failed"); + return; + } + } + + let mut file_picker = match FILE_PICKER.write() { + Ok(g) => g, + Err(e) => { + ::tracing::debug!(?e, "track_access: file picker write lock poisoned"); + return; + } + }; + let Some(ref mut picker) = *file_picker else { + return; + }; + + let frecency_guard = match FRECENCY.read() { + Ok(g) => g, + Err(e) => { + ::tracing::debug!(?e, "track_access: frecency read lock poisoned on update"); + return; + } + }; + let Some(ref frecency) = *frecency_guard else { + return; + }; + if let Err(e) = picker.update_single_file_frecency(&file_path, frecency) { + ::tracing::debug!( + ?e, + ?file_path, + "track_access: update_single_file_frecency failed" + ); + } + }); + + Ok(true) +} + +pub fn get_scan_progress(lua: &Lua, _: ()) -> LuaResult { + let file_picker = FILE_PICKER.read().into_lua_result()?; + let picker = file_picker + .as_ref() + .ok_or(Error::FilePickerMissing) + .into_lua_result()?; + let progress = picker.get_scan_progress(); + + let table = lua.create_table()?; + table.set("scanned_files_count", progress.scanned_files_count)?; + table.set("is_scanning", progress.is_scanning)?; + Ok(LuaValue::Table(table)) +} + +pub fn is_scanning(_: &Lua, _: ()) -> LuaResult { + let file_picker = FILE_PICKER.read().into_lua_result()?; + let picker = file_picker + .as_ref() + .ok_or(Error::FilePickerMissing) + .into_lua_result()?; + Ok(picker.is_scan_active()) +} + +pub fn get_git_root(_: &Lua, _: ()) -> LuaResult> { + let file_picker = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker else { + return Ok(None); + }; + + Ok(picker.git_root().map(|p| p.to_string_lossy().into_owned())) +} + +pub fn refresh_git_status(_: &Lua, _: ()) -> LuaResult { + FILE_PICKER.refresh_git_status(&FRECENCY).into_lua_result() +} + +pub fn get_file_access_count(_: &Lua, file_path: String) -> LuaResult { + let path = PathBuf::from(&file_path); + let frecency_guard = FRECENCY.read().into_lua_result()?; + let Some(ref frecency) = *frecency_guard else { + return Ok(0); + }; + let count = frecency.access_count(&path).into_lua_result()?; + Ok(count as u64) +} + +pub fn update_single_file_frecency(_: &Lua, file_path: String) -> LuaResult { + let frecency_guard = FRECENCY.read().into_lua_result()?; + let Some(ref frecency) = *frecency_guard else { + return Ok(false); + }; + + let mut file_picker = FILE_PICKER.write().into_lua_result()?; + let Some(ref mut picker) = *file_picker else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + + picker + .update_single_file_frecency(&file_path, frecency) + .into_lua_result()?; + Ok(true) +} + +pub fn stop_background_monitor(_: &Lua, _: ()) -> LuaResult { + // `stop_background_monitor` is non-blocking — the debouncer / + // owner threads exit on their next iteration, so it is safe to + // call under the FILE_PICKER write lock. + let mut file_picker = FILE_PICKER.write().into_lua_result()?; + let Some(ref mut picker) = *file_picker else { + return Err(error::to_lua_error(Error::FilePickerMissing)); + }; + picker.stop_background_monitor(); + Ok(true) +} + +pub fn cleanup_file_picker(_: &Lua, _: ()) -> LuaResult { + let mut file_picker = FILE_PICKER.write().into_lua_result()?; + if let Some(picker) = file_picker.take() { + drop(picker); + ::tracing::info!("FilePicker cleanup completed"); + Ok(true) + } else { + Ok(false) + } +} + +pub fn cancel_scan(_: &Lua, _: ()) -> LuaResult { + Ok(true) +} + +pub fn track_query_completion(_: &Lua, (query, file_path): (String, String)) -> LuaResult { + // Everything runs on a background thread — including the + // `FILE_PICKER.read()` used to fetch `project_path`. Doing that read + // on the main (UI) thread stalled nvim whenever a reindex writer was + // in flight: parking_lot's fair queue makes a read block behind a + // pending writer, so the main thread would freeze for the full + // duration of the scan. + let query_tracker = QUERY_TRACKER.clone(); + std::thread::spawn(move || { + let project_path = match FILE_PICKER.read() { + Ok(guard) => match *guard { + Some(ref picker) => picker.base_path().to_path_buf(), + None => return, + }, + Err(_) => return, + }; + + let file_path = match fff::path_utils::canonicalize(&file_path) { + Ok(path) => path, + Err(e) => { + tracing::warn!(?file_path, error = ?e, "Failed to canonicalize file path for tracking"); + return; + } + }; + + if let Ok(mut guard) = query_tracker.write() + && let Some(tracker) = guard.as_mut() + && let Err(e) = tracker.track_query_completion(&query, &project_path, &file_path) + { + tracing::error!( + query = %query, + file = %file_path.display(), + error = ?e, + "Failed to track query completion" + ); + } + }); + + Ok(true) +} + +pub fn get_historical_query(_: &Lua, offset: usize) -> LuaResult> { + let project_path = { + let file_picker = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker else { + return Ok(None); + }; + picker.base_path().to_path_buf() + }; + + let query_tracker = QUERY_TRACKER.read().into_lua_result()?; + let Some(ref tracker) = *query_tracker else { + return Ok(None); + }; + + tracker + .get_historical_query(&project_path, offset) + .into_lua_result() +} + +pub fn track_grep_query(_: &Lua, query: String) -> LuaResult { + // Move the `FILE_PICKER.read()` into the spawned worker too — + // see the note on `track_query_completion` above for why. + let query_tracker = QUERY_TRACKER.clone(); + std::thread::spawn(move || { + let project_path = match FILE_PICKER.read() { + Ok(guard) => match *guard { + Some(ref picker) => picker.base_path().to_path_buf(), + None => return, + }, + Err(_) => return, + }; + + if let Ok(mut guard) = query_tracker.write() + && let Some(ref mut tracker) = *guard + && let Err(e) = tracker.track_grep_query(&query, &project_path) + { + tracing::error!( + query = %query, + error = ?e, + "Failed to track grep query" + ); + } + }); + + Ok(true) +} + +pub fn get_historical_grep_query(_: &Lua, offset: usize) -> LuaResult> { + let project_path = { + let file_picker = FILE_PICKER.read().into_lua_result()?; + let Some(ref picker) = *file_picker else { + return Ok(None); + }; + picker.base_path().to_path_buf() + }; + + let query_tracker = QUERY_TRACKER.read().into_lua_result()?; + let Some(ref tracker) = *query_tracker else { + return Ok(None); + }; + + tracker + .get_historical_grep_query(&project_path, offset) + .into_lua_result() +} + +/// Parse a grep query string and return its text portion (with constraints stripped). +/// +/// Uses the Neovim grep parser config so filename-constraint detection follows +/// the user's setting, keeping Lua from re-implementing constraint detection. +pub fn parse_grep_query(lua: &Lua, query: String) -> LuaResult { + let parser = QueryParser::new(NvimGrepConfig); + let parsed = parser.parse(&query); + let table = lua.create_table()?; + table.set("grep_text", parsed.grep_text())?; + Ok(table) +} + +pub fn wait_for_initial_scan(_: &Lua, timeout_ms: Option) -> LuaResult { + let scanned = FILE_PICKER.wait_for_scan(Duration::from_millis(timeout_ms.unwrap_or(500))); + + Ok(scanned) +} + +pub fn init_tracing( + _: &Lua, + (log_file_path, log_level, retain_runs): (String, Option, Option), +) -> LuaResult { + crate::log::init_tracing(&log_file_path, log_level.as_deref(), retain_runs) + .map_err(|e| LuaError::RuntimeError(format!("Failed to initialize tracing: {}", e))) +} + +/// Returns health check information including version, git2 status, and repository detection +pub fn health_check(lua: &Lua, test_path: Option) -> LuaResult { + let table = lua.create_table()?; + table.set("version", env!("CARGO_PKG_VERSION"))?; + + let test_path = test_path + .map(PathBuf::from) + .unwrap_or_else(|| std::env::current_dir().unwrap_or_default()); + + let git_info = lua.create_table()?; + let git_version = git2::Version::get(); + let (major, minor, rev) = git_version.libgit2_version(); + let libgit2_version_str = format!("{}.{}.{}", major, minor, rev); + + match git2::Repository::discover(&test_path) { + Ok(repo) => { + git_info.set("available", true)?; + git_info.set("repository_found", true)?; + if let Some(workdir) = repo.workdir() { + git_info.set("workdir", workdir.to_string_lossy().to_string())?; + } + // Get git2 version info + git_info.set("libgit2_version", libgit2_version_str.clone())?; + } + Err(e) => { + git_info.set("available", true)?; + git_info.set("repository_found", false)?; + git_info.set("error", e.message().to_string())?; + git_info.set("libgit2_version", libgit2_version_str)?; + } + } + table.set("git", git_info)?; + + // Check file picker status + let picker_info = lua.create_table()?; + match FILE_PICKER.read() { + Ok(guard) => { + if let Some(ref picker) = *guard { + picker_info.set("initialized", true)?; + picker_info.set( + "base_path", + picker.base_path().to_string_lossy().to_string(), + )?; + picker_info.set("is_scanning", picker.is_scan_active())?; + let progress = picker.get_scan_progress(); + picker_info.set("indexed_files", progress.scanned_files_count)?; + } else { + picker_info.set("initialized", false)?; + } + } + Err(_) => { + picker_info.set("initialized", false)?; + picker_info.set("error", "Failed to acquire file picker lock")?; + } + } + table.set("file_picker", picker_info)?; + + let frecency_info = lua.create_table()?; + match FRECENCY.read() { + Ok(guard) => { + frecency_info.set("initialized", guard.is_some())?; + + if let Some(ref frecency) = *guard { + match frecency.get_health() { + Ok(health) => { + let healthcheck_table = lua.create_table()?; + healthcheck_table.set("path", health.path)?; + healthcheck_table.set("disk_size", health.disk_size)?; + healthcheck_table.set("healthy", health.healthy)?; + for (name, count) in health.entry_counts { + healthcheck_table.set(name, count)?; + } + frecency_info.set("db_healthcheck", healthcheck_table)?; + } + Err(e) => { + frecency_info.set("db_healthcheck_error", e.to_string())?; + } + } + } + } + Err(_) => { + frecency_info.set("initialized", false)?; + frecency_info.set("error", "Failed to acquire frecency lock")?; + } + } + table.set("frecency", frecency_info)?; + + let query_tracker_info = lua.create_table()?; + match QUERY_TRACKER.read() { + Ok(guard) => { + query_tracker_info.set("initialized", guard.is_some())?; + if let Some(ref query_history) = *guard { + match query_history.get_health() { + Ok(health) => { + let healthcheck_table = lua.create_table()?; + healthcheck_table.set("path", health.path)?; + healthcheck_table.set("disk_size", health.disk_size)?; + healthcheck_table.set("healthy", health.healthy)?; + for (name, count) in health.entry_counts { + healthcheck_table.set(name, count)?; + } + query_tracker_info.set("db_healthcheck", healthcheck_table)?; + } + Err(e) => { + query_tracker_info.set("db_healthcheck_error", e.to_string())?; + } + } + } + } + Err(_) => { + query_tracker_info.set("initialized", false)?; + query_tracker_info.set("error", "Failed to acquire query tracker lock")?; + } + } + table.set("query_tracker", query_tracker_info)?; + + Ok(LuaValue::Table(table)) +} + +pub fn shorten_path( + _: &Lua, + (path, max_size, strategy): (String, usize, Option), +) -> LuaResult { + let strategy = strategy + .map(|v| -> LuaResult { + match v { + mlua::Value::String(ref s) => { + let name = s + .to_str() + .map(|s| s.to_owned()) + .unwrap_or_else(|_| "middle_number".to_string()); + Ok(PathShortenStrategy::from_name(&name)) + } + _ => Ok(PathShortenStrategy::default()), + } + }) + .transpose()? + .unwrap_or_default(); + + shorten_path_with_cache(strategy, max_size, Path::new(&path)).map_err(LuaError::RuntimeError) +} + +fn create_exports(lua: &Lua) -> LuaResult { + let exports = lua.create_table()?; + exports.set("init_db", lua.create_function(init_db)?)?; + exports.set( + "destroy_frecency_db", + lua.create_function(destroy_frecency_db)?, + )?; + exports.set("init_file_picker", lua.create_function(init_file_picker)?)?; + exports.set( + "restart_index_in_path", + lua.create_function(restart_index_in_path)?, + )?; + exports.set("scan_files", lua.create_function(scan_files)?)?; + exports.set( + "fuzzy_search_files", + lua.create_function(fuzzy_search_files)?, + )?; + exports.set( + "fuzzy_search_directories", + lua.create_function(fuzzy_search_directories)?, + )?; + exports.set( + "fuzzy_search_mixed", + lua.create_function(fuzzy_search_mixed)?, + )?; + exports.set("live_grep", lua.create_function(live_grep)?)?; + exports.set("track_access", lua.create_function(track_access)?)?; + exports.set( + "get_file_access_count", + lua.create_function(get_file_access_count)?, + )?; + exports.set("cancel_scan", lua.create_function(cancel_scan)?)?; + exports.set("get_scan_progress", lua.create_function(get_scan_progress)?)?; + exports.set( + "refresh_git_status", + lua.create_function(refresh_git_status)?, + )?; + exports.set("get_git_root", lua.create_function(get_git_root)?)?; + exports.set( + "stop_background_monitor", + lua.create_function(stop_background_monitor)?, + )?; + exports.set("init_tracing", lua.create_function(init_tracing)?)?; + exports.set( + "wait_for_initial_scan", + lua.create_function(wait_for_initial_scan)?, + )?; + exports.set( + "cleanup_file_picker", + lua.create_function(cleanup_file_picker)?, + )?; + exports.set("destroy_query_db", lua.create_function(destroy_query_db)?)?; + exports.set( + "track_query_completion", + lua.create_function(track_query_completion)?, + )?; + exports.set( + "get_historical_query", + lua.create_function(get_historical_query)?, + )?; + exports.set("track_grep_query", lua.create_function(track_grep_query)?)?; + exports.set( + "get_historical_grep_query", + lua.create_function(get_historical_grep_query)?, + )?; + exports.set("health_check", lua.create_function(health_check)?)?; + exports.set("shorten_path", lua.create_function(shorten_path)?)?; + exports.set("hex_dump", lua.create_function(hex_dump::hex_dump)?)?; + exports.set("parse_grep_query", lua.create_function(parse_grep_query)?)?; + + Ok(exports) +} + +// https://github.com/mlua-rs/mlua/issues/318 +#[mlua::lua_module(skip_memory_check)] +fn fff_nvim(lua: &Lua) -> LuaResult { + // Install panic hook + SIGSEGV chain handler IMMEDIATELY on module load. + // Without this, a crash inside fff produces a silent nvim death — neovim's + // own handler does not log a Rust-side backtrace. + crate::log::install_panic_hook(); + + create_exports(lua) +} diff --git a/crates/fff-nvim/src/log.rs b/crates/fff-nvim/src/log.rs new file mode 100644 index 0000000..3c1a5c9 --- /dev/null +++ b/crates/fff-nvim/src/log.rs @@ -0,0 +1,3 @@ +//! Logging setup for fff-nvim — delegates to the shared fff-core::log utilities. + +pub use fff::log::{init_tracing, install_panic_hook}; diff --git a/crates/fff-nvim/src/lua_types.rs b/crates/fff-nvim/src/lua_types.rs new file mode 100644 index 0000000..83ef10a --- /dev/null +++ b/crates/fff-nvim/src/lua_types.rs @@ -0,0 +1,281 @@ +use fff::file_picker::FilePicker; +use fff::git::format_git_status; +use fff::{ + DirItem, DirSearchResult, FileItem, GrepResult, Location, MixedItemRef, MixedSearchResult, + Score, SearchResult, +}; +use mlua::prelude::*; + +pub struct SearchResultLua<'a> { + inner: SearchResult<'a>, + picker: &'a FilePicker, +} + +impl<'a> SearchResultLua<'a> { + pub fn new(inner: SearchResult<'a>, picker: &'a FilePicker) -> Self { + Self { inner, picker } + } +} + +pub struct GrepResultLua<'a> { + inner: GrepResult<'a>, + picker: &'a FilePicker, +} + +impl<'a> GrepResultLua<'a> { + pub fn new(inner: GrepResult<'a>, picker: &'a FilePicker) -> Self { + Self { inner, picker } + } +} + +pub struct DirSearchResultLua<'a> { + inner: DirSearchResult<'a>, + picker: &'a FilePicker, +} + +impl<'a> DirSearchResultLua<'a> { + pub fn new(inner: DirSearchResult<'a>, picker: &'a FilePicker) -> Self { + Self { inner, picker } + } +} + +pub struct MixedSearchResultLua<'a> { + inner: MixedSearchResult<'a>, + picker: &'a FilePicker, +} + +impl<'a> MixedSearchResultLua<'a> { + pub fn new(inner: MixedSearchResult<'a>, picker: &'a FilePicker) -> Self { + Self { inner, picker } + } +} + +struct LuaPosition((i32, i32)); + +impl IntoLua for LuaPosition { + fn into_lua(self, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + table.set("line", self.0.0)?; + table.set("col", self.0.1)?; + Ok(LuaValue::Table(table)) + } +} + +fn file_item_into_lua(item: &FileItem, lua: &Lua, picker: &FilePicker) -> LuaResult { + let table = lua.create_table()?; + table.set("type", "file")?; + table.set("relative_path", item.relative_path(picker))?; + table.set("name", item.file_name(picker))?; + table.set("size", item.size)?; + table.set("modified", item.modified)?; + table.set("access_frecency_score", item.access_frecency_score)?; + table.set( + "modification_frecency_score", + item.modification_frecency_score, + )?; + table.set("total_frecency_score", item.total_frecency_score())?; + table.set("git_status", format_git_status(item.git_status))?; + table.set("is_binary", item.is_binary())?; + Ok(LuaValue::Table(table)) +} + +fn dir_item_into_lua(item: &DirItem, lua: &Lua, picker: &FilePicker) -> LuaResult { + let table = lua.create_table()?; + let name = item + .dir_name(picker) + .trim_end_matches(std::path::MAIN_SEPARATOR) + .trim_end_matches('/') + .to_owned(); + table.set("type", "directory")?; + table.set("relative_path", item.relative_path(picker))?; + table.set("name", name)?; + table.set("max_access_frecency", item.max_access_frecency())?; + Ok(LuaValue::Table(table)) +} + +fn score_into_lua(score: &Score, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + table.set("total", score.total)?; + table.set("base_score", score.base_score)?; + table.set("filename_bonus", score.filename_bonus)?; + table.set("special_filename_bonus", score.special_filename_bonus)?; + table.set("frecency_boost", score.frecency_boost)?; + table.set("distance_penalty", score.distance_penalty)?; + table.set("current_file_penalty", score.current_file_penalty)?; + table.set("combo_match_boost", score.combo_match_boost)?; + table.set("path_alignment_bonus", score.path_alignment_bonus)?; + table.set("match_type", score.match_type)?; + table.set("exact_match", score.exact_match)?; + Ok(LuaValue::Table(table)) +} + +fn location_into_lua(location: &Location, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + match location { + Location::Line(line) => { + table.set("line", *line)?; + } + Location::Position { line, col } => { + table.set("line", *line)?; + table.set("col", *col)?; + } + Location::Range { start, end } => { + table.set("start", LuaPosition(*start))?; + table.set("end", LuaPosition(*end))?; + } + } + Ok(LuaValue::Table(table)) +} + +impl IntoLua for SearchResultLua<'_> { + fn into_lua(self, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + + // Convert items + let items_table = lua.create_table()?; + for (i, item) in self.inner.items.iter().enumerate() { + items_table.set(i + 1, file_item_into_lua(item, lua, self.picker)?)?; + } + table.set("items", items_table)?; + + // Convert scores + let scores_table = lua.create_table()?; + for (i, score) in self.inner.scores.iter().enumerate() { + scores_table.set(i + 1, score_into_lua(score, lua)?)?; + } + table.set("scores", scores_table)?; + + table.set("total_matched", self.inner.total_matched)?; + table.set("total_files", self.inner.total_files)?; + + if let Some(location) = &self.inner.location { + table.set("location", location_into_lua(location, lua)?)?; + } + + Ok(LuaValue::Table(table)) + } +} + +impl IntoLua for DirSearchResultLua<'_> { + fn into_lua(self, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + + let items_table = lua.create_table()?; + for (i, item) in self.inner.items.iter().enumerate() { + items_table.set(i + 1, dir_item_into_lua(item, lua, self.picker)?)?; + } + table.set("items", items_table)?; + + let scores_table = lua.create_table()?; + for (i, score) in self.inner.scores.iter().enumerate() { + scores_table.set(i + 1, score_into_lua(score, lua)?)?; + } + table.set("scores", scores_table)?; + + table.set("total_matched", self.inner.total_matched)?; + table.set("total_dirs", self.inner.total_dirs)?; + + Ok(LuaValue::Table(table)) + } +} + +impl IntoLua for MixedSearchResultLua<'_> { + fn into_lua(self, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + + let items_table = lua.create_table()?; + for (i, item) in self.inner.items.iter().enumerate() { + let lua_item = match item { + MixedItemRef::File(file) => file_item_into_lua(file, lua, self.picker)?, + MixedItemRef::Dir(dir) => dir_item_into_lua(dir, lua, self.picker)?, + }; + items_table.set(i + 1, lua_item)?; + } + table.set("items", items_table)?; + + let scores_table = lua.create_table()?; + for (i, score) in self.inner.scores.iter().enumerate() { + scores_table.set(i + 1, score_into_lua(score, lua)?)?; + } + table.set("scores", scores_table)?; + + table.set("total_matched", self.inner.total_matched)?; + table.set("total_files", self.inner.total_files)?; + table.set("total_dirs", self.inner.total_dirs)?; + + if let Some(location) = &self.inner.location { + table.set("location", location_into_lua(location, lua)?)?; + } + + Ok(LuaValue::Table(table)) + } +} + +impl IntoLua for GrepResultLua<'_> { + fn into_lua(self, lua: &Lua) -> LuaResult { + let table = lua.create_table()?; + + // Convert grep match items — each includes file metadata + match metadata + let items_table = lua.create_table()?; + for (i, m) in self.inner.matches.iter().enumerate() { + let item = lua.create_table()?; + + // File metadata from the deduplicated files vec + let file = self.inner.files[m.file_index]; + item.set("relative_path", file.relative_path(self.picker))?; + item.set("name", file.file_name(self.picker))?; + item.set("is_binary", file.is_binary())?; + item.set("git_status", format_git_status(file.git_status))?; + item.set("size", file.size)?; + item.set("modified", file.modified)?; + item.set("total_frecency_score", file.total_frecency_score())?; + item.set("access_frecency_score", file.access_frecency_score)?; + item.set( + "modification_frecency_score", + file.modification_frecency_score, + )?; + + // Match metadata + item.set("line_number", m.line_number)?; + item.set("col", m.col)?; + item.set("byte_offset", m.byte_offset)?; + + // There is a little race window when fff can return matches inside of a non-binary + // classified entities, the window is minimal but it errors out neovim so guard it + let is_binary_content = m.line_content.as_bytes().contains(&0u8); + item.set("is_binary_content", is_binary_content)?; + item.set("line_content", m.line_content.as_str())?; + + // Match byte ranges within line_content + let ranges = lua.create_table()?; + for (j, &(start, end)) in m.match_byte_offsets.iter().enumerate() { + let range = lua.create_table()?; + range.set(1, start)?; + range.set(2, end)?; + ranges.set(j + 1, range)?; + } + item.set("match_ranges", ranges)?; + + // Fuzzy match score (only set in fuzzy grep mode, nil otherwise) + if let Some(score) = m.fuzzy_score { + item.set("fuzzy_score", score)?; + } + + items_table.set(i + 1, item)?; + } + table.set("items", items_table)?; + + table.set("total_matched", self.inner.matches.len())?; + table.set("total_files_searched", self.inner.total_files_searched)?; + table.set("total_files", self.inner.total_files)?; + table.set("filtered_file_count", self.inner.filtered_file_count)?; + table.set("next_file_offset", self.inner.next_file_offset)?; + + // Pass regex fallback error to Lua (nil if no error) + if let Some(ref err) = self.inner.regex_fallback_error { + table.set("regex_fallback_error", err.as_str())?; + } + + Ok(LuaValue::Table(table)) + } +} diff --git a/crates/fff-nvim/src/path_shortening.rs b/crates/fff-nvim/src/path_shortening.rs new file mode 100644 index 0000000..2252cd1 --- /dev/null +++ b/crates/fff-nvim/src/path_shortening.rs @@ -0,0 +1,745 @@ +//! Path shortening utilities for display in Neovim UI +//! +//! This module provides functionality to shorten file paths for display +//! in the picker UI with various strategies. + +use once_cell::sync::Lazy; +use std::borrow::Cow; +use std::path::{Component, MAIN_SEPARATOR, Path, PathBuf}; +use std::sync::RwLock; + +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub enum PathShortenStrategy { + #[default] + MiddleNumber, + Middle, + End, + Start, +} + +struct CacheEntry { + shortened: String, + max_size: usize, + strategy: PathShortenStrategy, +} + +struct PathCache { + map: ahash::AHashMap, + max_entries: usize, +} + +impl PathCache { + fn new(max_entries: usize) -> Self { + Self { + map: ahash::AHashMap::with_capacity(max_entries), + max_entries, + } + } + + #[tracing::instrument(skip(self), fields(path = %path.display(), max_size), level = tracing::Level::TRACE)] + fn get(&self, path: &Path, max_size: usize, strategy: PathShortenStrategy) -> Option<&str> { + self.map.get(path).and_then(|entry| { + // Only return cached value if both max_size and strategy match + if entry.max_size == max_size && entry.strategy == strategy { + Some(entry.shortened.as_str()) + } else { + None + } + }) + } + + fn insert( + &mut self, + path: PathBuf, + shortened: String, + max_size: usize, + strategy: PathShortenStrategy, + ) { + // Simple eviction: clear half the cache when full + if self.map.len() >= self.max_entries { + let keys_to_remove: Vec<_> = self + .map + .keys() + .take(self.max_entries / 2) + .cloned() + .collect(); + for key in keys_to_remove { + self.map.remove(&key); + } + } + self.map.insert( + path, + CacheEntry { + shortened, + max_size, + strategy, + }, + ); + } +} + +// this is the amount of PATHS not entries +const DEFAULT_CACHE_SIZE: usize = 8192; + +static PATH_SHORTEN_CACHE: Lazy> = + Lazy::new(|| RwLock::new(PathCache::new(DEFAULT_CACHE_SIZE))); + +pub fn shorten_path_with_cache( + strategy: PathShortenStrategy, + max_size: usize, + path: &Path, +) -> Result { + { + let cache = PATH_SHORTEN_CACHE + .read() + .map_err(|_| "Failed to acquire path cache lock".to_string())?; + if let Some(cached) = cache.get(path, max_size, strategy) { + tracing::trace!("Cache hit for path '{}'", path.display()); + return Ok(cached.to_string()); + } + } + + let shortened = strategy.shorten_path(path, max_size); + { + let mut cache = PATH_SHORTEN_CACHE + .write() + .map_err(|_| "Failed to acquire path cache lock".to_string())?; + cache.insert(path.to_path_buf(), shortened.clone(), max_size, strategy); + } + + Ok(shortened) +} + +impl PathShortenStrategy { + /// Parse a strategy from a string name + pub fn from_name(name: &str) -> Self { + match name { + "middle_number" => PathShortenStrategy::MiddleNumber, + "middle" => PathShortenStrategy::Middle, + "end" => PathShortenStrategy::End, + "start" => PathShortenStrategy::Start, + _ => PathShortenStrategy::MiddleNumber, + } + } +} + +impl PathShortenStrategy { + pub fn shorten_path(&self, path: &Path, max_size: usize) -> String { + const MIN_SMART_SHORTEN_SIZE: usize = 8; + + let sep = MAIN_SEPARATOR; + + let path_str = path.to_string_lossy(); + if path_str.len() <= max_size { + return path_str.to_string(); + } + + // If max_size is too small for smart shortening, just truncate + if max_size < MIN_SMART_SHORTEN_SIZE { + return match self { + PathShortenStrategy::Start => Self::truncate_str_keep_end(&path_str, max_size), + _ => Self::truncate_str(&path_str, max_size), + }; + } + + let components: Vec<&str> = path + .components() + .filter_map(|c| match c { + Component::Normal(s) => s.to_str(), + _ => None, + }) + .collect(); + + if components.is_empty() { + return path_str.to_string(); + } + + // For single component, just truncate it + if components.len() == 1 { + return match self { + PathShortenStrategy::Start => Self::truncate_str_keep_end(components[0], max_size), + _ => Self::truncate_str(components[0], max_size), + }; + } + + match self { + PathShortenStrategy::End => { + // Simple truncation from the end + let mut result = String::new(); + + for (i, component) in components.iter().enumerate() { + let candidate = if i == 0 { + component.to_string() + } else { + format!("{}{}{}", result, sep, component) + }; + + if candidate.len() <= max_size { + result = candidate; + } else { + break; + } + } + + // If even the first component is too long, truncate it + if result.is_empty() && !components.is_empty() { + return components.first().map_or(String::new(), |component| { + let mut component = component.to_string(); + + component.truncate(max_size); + component + }); + } + + result + } + PathShortenStrategy::Middle | PathShortenStrategy::MiddleNumber => { + let use_number = matches!(self, PathShortenStrategy::MiddleNumber); + self.shorten_middle(&components, max_size, use_number, sep) + } + PathShortenStrategy::Start => Self::shorten_start(&components, max_size, sep), + } + } + + // rust doesn't have an ergonomic way to clone and truncate + fn truncate_str(s: &str, max_len: usize) -> String { + if max_len == 0 { + return String::new(); + } + + let char_count = s.chars().count(); + if char_count <= max_len { + return s.to_string(); + } + + // Just take the first max_len characters - no ".." suffix + s.chars().take(max_len).collect() + } + + fn truncate_str_keep_end(s: &str, max_len: usize) -> String { + if max_len == 0 { + return String::new(); + } + + let char_count = s.chars().count(); + if char_count <= max_len { + return s.to_string(); + } + + s.chars().skip(char_count - max_len).collect() + } + + // e.g. ".../parts/ai_extracted.rs" + fn shorten_start(components: &[&str], max_size: usize, sep: char) -> String { + let total = components.len(); + let last = components[total - 1]; + + // Try smallest hidden_count first to maximize shown context. + for hidden in 1..total { + let right_parts = &components[hidden..]; + let ellipsis = Self::make_ellipsis(hidden, false); + + let right_content_len: usize = right_parts.iter().map(|s| s.len()).sum(); + let right_seps = right_parts.len().saturating_sub(1); + let total_len = ellipsis.len() + 1 + right_content_len + right_seps; + + if total_len <= max_size { + let mut result = String::with_capacity(total_len); + result.push_str(&ellipsis); + result.push(sep); + for (i, part) in right_parts.iter().enumerate() { + if i > 0 { + result.push(sep); + } + result.push_str(part); + } + return result; + } + } + + // Even ellipsis + last won't fit: show the end of the last component. + Self::truncate_str_keep_end(last, max_size) + } + + fn shorten_middle( + &self, + components: &[&str], + max_size: usize, + use_number: bool, + sep: char, + ) -> String { + let total = components.len(); + + // For 2 components, just show both or truncate to fit + if total <= 2 { + let joined = components.join(&sep.to_string()); + if joined.len() <= max_size { + return joined; + } + // Try to keep last intact, truncate first + let last = components[total - 1]; + let available_for_first = max_size.saturating_sub(1 + last.len()); // sep + last + if available_for_first > 0 && last.len() < max_size { + let truncated = Self::truncate_str(components[0], available_for_first); + let mut result = String::with_capacity(truncated.len() + 1 + last.len()); + result.push_str(&truncated); + result.push(sep); + result.push_str(last); + return result; + } + // Last component alone exceeds max_size, must truncate it + return Self::truncate_str(last, max_size); + } + + let first = components[0]; + let last = components[total - 1]; + + let initial_hidden = total - 2; + let ellipsis = Self::make_ellipsis(initial_hidden, use_number); + + // Minimum pattern: first/.../last + let min_overhead = 2 + ellipsis.len(); // two separators + ellipsis + let min_content = first.len() + last.len(); + + if min_content + min_overhead <= max_size { + // We can fit first/.../last, now try to add more components + return self.expand_middle(components, max_size, use_number, sep); + } + + // Need to truncate to fit max_size + // Priority: keep last intact if possible, truncate first, then truncate last if needed + let needed_for_last = last.len() + 1 + ellipsis.len() + 1; // sep + ellipsis + sep + last + if needed_for_last <= max_size { + let available_for_first = max_size - needed_for_last; + let truncated_first = Self::truncate_str(first, available_for_first); + let ellipsis = Self::make_ellipsis(initial_hidden, use_number); + // truncated_first + sep + ellipsis + sep + last + let capacity = truncated_first.len() + 1 + ellipsis.len() + 1 + last.len(); + let mut result = String::with_capacity(capacity); + result.push_str(&truncated_first); + result.push(sep); + result.push_str(&ellipsis); + result.push(sep); + result.push_str(last); + return result; + } + + let needed_for_ellipsis_last = ellipsis.len() + 1 + last.len(); // ellipsis + sep + last + if needed_for_ellipsis_last <= max_size { + let mut result = String::with_capacity(needed_for_ellipsis_last); + result.push_str(&ellipsis); + result.push(sep); + result.push_str(last); + return result; + } + + // Can't fit ellipsis + last, just show as much of last as possible + Self::truncate_str(last, max_size) + } + + fn expand_middle( + &self, + components: &[&str], + max_size: usize, + use_number: bool, + sep: char, + ) -> String { + let total = components.len(); + + // Start with minimum: first/...or..N../last + let mut left_end = 1; // exclusive index for left components + let mut right_start = total - 1; // inclusive index for right components + + // Try to add more components from both sides + loop { + if right_start <= left_end { + break; + } + + let mut added = false; + + // Try adding from RIGHT first (to show more context near the file) + if right_start > left_end + 1 { + let hidden = right_start - 1 - left_end; + let candidate = Self::build_middle_result( + components, + left_end, + right_start - 1, + hidden, + use_number, + sep, + ); + if candidate.len() <= max_size { + right_start -= 1; + added = true; + } + } + + // Try adding from LEFT + if left_end < right_start - 1 { + let hidden = right_start - (left_end + 1); + let candidate = Self::build_middle_result( + components, + left_end + 1, + right_start, + hidden, + use_number, + sep, + ); + if candidate.len() <= max_size { + left_end += 1; + added = true; + } + } + + if !added { + break; + } + } + + let hidden = right_start - left_end; + Self::build_middle_result(components, left_end, right_start, hidden, use_number, sep) + } + + fn build_middle_result( + components: &[&str], + left_end: usize, + right_start: usize, + hidden_count: usize, + use_number: bool, + sep: char, + ) -> String { + let ellipsis = Self::make_ellipsis(hidden_count, use_number); + + let left_parts = &components[..left_end]; + let right_parts = &components[right_start..]; + + // Pre-calculate capacity + let left_len: usize = left_parts.iter().map(|s| s.len()).sum(); + let right_len: usize = right_parts.iter().map(|s| s.len()).sum(); + let left_seps = if left_parts.is_empty() { + 0 + } else { + left_parts.len() - 1 + }; + let right_seps = if right_parts.is_empty() { + 0 + } else { + right_parts.len() - 1 + }; + // +2 for separators around ellipsis (or +1 if left is empty) + let extra_seps = if left_parts.is_empty() { 1 } else { 2 }; + let capacity = left_len + right_len + left_seps + right_seps + ellipsis.len() + extra_seps; + + let mut result = String::with_capacity(capacity); + + // Build left part + for (i, part) in left_parts.iter().enumerate() { + if i > 0 { + result.push(sep); + } + result.push_str(part); + } + + // Add separator before ellipsis (only if left is not empty) + if !left_parts.is_empty() { + result.push(sep); + } + + // Add ellipsis + result.push_str(&ellipsis); + + // Add separator after ellipsis + result.push(sep); + + // Build right part + for (i, part) in right_parts.iter().enumerate() { + if i > 0 { + result.push(sep); + } + result.push_str(part); + } + + result + } + + fn make_ellipsis(hidden_count: usize, use_number: bool) -> Cow<'static, str> { + match hidden_count { + 1 => ".".into(), + 2 => "..".into(), + 3 if use_number => "...".into(), + n if use_number => format!(".{}.", n).into(), + _ => "...".into(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_path_shorten_strategy_middle() { + // Test with directory paths (not file paths) - this is what Lua passes + let path = Path::new("core_workflow_service/db/model/parts/ai_extracted"); + + // With 25 chars, first component must be truncated + // "core_workflow_service" is 21 chars, so we need to truncate it + let shortened = PathShortenStrategy::Middle.shorten_path(path, 25); + assert!( + shortened.len() <= 25, + "Result '{}' should be <= 25 chars", + shortened + ); + assert!(shortened.contains("..."), "Should contain ellipsis"); + assert!( + shortened.ends_with("ai_extracted"), + "Should end with last component" + ); + + // With 45 chars, can fit more without truncation + let shortened = PathShortenStrategy::Middle.shorten_path(path, 45); + assert!(shortened.len() <= 45); + assert!(shortened.starts_with("core_workflow_service")); + + // Shorter path that fits better + let path2 = Path::new("src/components/ui/buttons"); + let shortened = PathShortenStrategy::Middle.shorten_path(path2, 20); + assert!( + shortened.len() <= 20, + "Result '{}' should be <= 20 chars", + shortened + ); + + // Very small max_size - should still produce something reasonable + let shortened = PathShortenStrategy::Middle.shorten_path(path2, 10); + assert!( + shortened.len() <= 10, + "Result '{}' should be <= 10 chars", + shortened + ); + } + + #[test] + fn test_path_shroten_strategy_middle_number() { + // Test with directory paths (not file paths) + // middle_number uses dots for 1-3 hidden, numbers for 4+ + + // Path with only 2 hidden segments - should use dots + let path = Path::new("core_workflow_service/graphql/types/parts"); + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path, 40); + assert!( + shortened.len() <= 40, + "Result '{}' should be <= 40 chars", + shortened + ); + // With only 2 hidden, should use dots not numbers + assert!( + shortened.contains('.'), + "Should contain dots, got '{}'", + shortened + ); + + // Path with many segments, small space - should use .N. format when 4+ hidden + let path2 = Path::new("a/b/c/d/e/f/g/h/i/j"); + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path2, 12); + assert!( + shortened.len() <= 12, + "Result '{}' should be <= 12 chars", + shortened + ); + // With 8 hidden (showing only a and j), should show number + assert!( + shortened.contains('.') && shortened.chars().any(|c| c.is_ascii_digit()), + "Should contain .N. pattern for 4+ hidden, got '{}'", + shortened + ); + + // Very small max_size + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path2, 5); + assert!( + shortened.len() <= 5, + "Result '{}' should be <= 5 chars", + shortened + ); + } + + #[test] + fn test_path_shroten_strategy_end() { + let path = Path::new("core_workflow_service/db/model/parts/ai_extracted"); + let shortened = PathShortenStrategy::End.shorten_path(path, 25); + assert!(shortened.len() <= 25); + assert!(shortened.starts_with("core_workflow_service")); + + // Shorter constraint - truncates first component + let shortened = PathShortenStrategy::End.shorten_path(path, 15); + assert!( + shortened.len() <= 15, + "Result '{}' should be <= 15 chars", + shortened + ); + } + + #[test] + fn test_path_shorten_strategy_start() { + let path = Path::new("core_workflow_service/db/model/parts/ai_extracted"); + + // Ample space - full path fits + let shortened = PathShortenStrategy::Start.shorten_path(path, 100); + assert_eq!( + shortened, + "core_workflow_service/db/model/parts/ai_extracted" + ); + + // 25 chars - should drop first components, keep tail intact + let shortened = PathShortenStrategy::Start.shorten_path(path, 25); + assert!( + shortened.len() <= 25, + "Result '{}' should be <= 25 chars", + shortened + ); + assert!( + shortened.ends_with("ai_extracted"), + "Should preserve last component, got '{}'", + shortened + ); + assert!( + shortened.starts_with('.'), + "Should start with ellipsis, got '{}'", + shortened + ); + + // Tight space where ellipsis + last won't fit: should still end with + // (a suffix of) the last component, never any leading content. + let shortened = PathShortenStrategy::Start.shorten_path(path, 8); + assert!(shortened.len() <= 8); + assert!( + "ai_extracted".ends_with(shortened.as_str()), + "Tight start shortening must keep end of last component, got '{}'", + shortened + ); + + // Single long component falls back to keeping the end + let path2 = Path::new("very_long_directory_name_with_many_chars"); + let shortened = PathShortenStrategy::Start.shorten_path(path2, 10); + assert_eq!(shortened.len(), 10); + assert!( + "very_long_directory_name_with_many_chars".ends_with(shortened.as_str()), + "Should keep end of single component, got '{}'", + shortened + ); + } + + #[test] + fn test_shorten_path_caching() { + let path = Path::new("home/user/projects/rust/project/src/components/ui"); + + // First call should compute and cache + let result1 = shorten_path_with_cache(PathShortenStrategy::MiddleNumber, 25, path).unwrap(); + + // Second call should hit cache + let result2 = shorten_path_with_cache(PathShortenStrategy::MiddleNumber, 25, path).unwrap(); + + assert_eq!(result1, result2); + + // Different max_size should produce different result (more space = longer result) + let result3 = shorten_path_with_cache(PathShortenStrategy::MiddleNumber, 50, path).unwrap(); + assert!( + result3.len() >= result1.len(), + "More space should allow longer result" + ); + + // Different strategy must NOT return the cached MiddleNumber result. + let result_start = shorten_path_with_cache(PathShortenStrategy::Start, 25, path).unwrap(); + let direct_start = PathShortenStrategy::Start.shorten_path(path, 25); + assert_eq!( + result_start, direct_start, + "Cache must be keyed on strategy, not return stale result" + ); + } + + #[test] + fn test_path_always_fits_max_size() { + // Path must ALWAYS fit within max_size - this is a strict requirement + let paths = [ + "core_workflow_service/db/model/parts/ai_extracted", + "home/user/projects/rust/project/src", + "a/b/c/d/e/f/g/h", + "very_long_directory_name/another_long_one/and_more", + ]; + + for path_str in paths { + let path = Path::new(path_str); + for max_size in [10, 15, 20, 25, 30, 40, 50] { + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path, max_size); + assert!( + shortened.len() <= max_size, + "Path '{}' with max_size {} produced '{}' ({} chars)", + path_str, + max_size, + shortened, + shortened.len() + ); + + let shortened = PathShortenStrategy::Middle.shorten_path(path, max_size); + assert!( + shortened.len() <= max_size, + "Path '{}' with max_size {} produced '{}' ({} chars)", + path_str, + max_size, + shortened, + shortened.len() + ); + + let shortened = PathShortenStrategy::Start.shorten_path(path, max_size); + assert!( + shortened.len() <= max_size, + "Path '{}' with max_size {} produced '{}' ({} chars)", + path_str, + max_size, + shortened, + shortened.len() + ); + } + } + } + + #[test] + fn test_small_max_size_simple_truncation() { + // When max_size is very small (< MIN_SMART_SHORTEN_SIZE), should just truncate + let path = Path::new("core_workflow_service/db/model/parts"); + + // With max_size=6, should just truncate (below threshold) + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path, 6); + assert_eq!(shortened.len(), 6); + assert_eq!(shortened, "core_w"); + + // With max_size=10, smart shortening kicks in + let shortened = PathShortenStrategy::Middle.shorten_path(path, 10); + assert!(shortened.len() <= 10); + } + + #[test] + fn test_prioritizes_last_component() { + // When space allows, last component should be shown in full + let path = Path::new("first/medium/last_component"); + + // With enough space, last component should be intact + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path, 25); + assert!( + shortened.ends_with("last_component"), + "Should preserve last component when space allows, got '{}'", + shortened + ); + assert!(shortened.len() <= 25); + + // When space is too tight, last component may be truncated to fit + let shortened = PathShortenStrategy::MiddleNumber.shorten_path(path, 10); + assert!( + shortened.len() <= 10, + "Must fit within max_size, got '{}' ({} chars)", + shortened, + shortened.len() + ); + } +} diff --git a/crates/fff-nvim/src/user_config.rs b/crates/fff-nvim/src/user_config.rs new file mode 100644 index 0000000..0c8d7e5 --- /dev/null +++ b/crates/fff-nvim/src/user_config.rs @@ -0,0 +1,47 @@ +use fff::{GrepConfig, ParserConfig}; +use std::sync::RwLock; + +#[derive(Debug, Clone, Copy, Default)] +pub struct UserConfigOptions { + pub enable_filename_constraint: bool, +} + +static USER_CONFIG: RwLock = RwLock::new(UserConfigOptions { + enable_filename_constraint: false, +}); + +pub fn set_global_user_config(config: UserConfigOptions) { + if let Ok(mut guard) = USER_CONFIG.write() { + *guard = config; + } +} + +fn get_global_user_config() -> UserConfigOptions { + USER_CONFIG.read().map(|c| *c).unwrap_or_default() +} + +/// Grep query parser config for Neovim configured by user +/// Same as `GrepConfig` but lets the user configure some behavior of query parsing +pub struct NvimGrepConfig; + +impl ParserConfig for NvimGrepConfig { + fn enable_path_segments(&self) -> bool { + true + } + + fn enable_git_status(&self) -> bool { + false + } + + fn enable_location(&self) -> bool { + false + } + + fn enable_filename_constraint(&self) -> bool { + get_global_user_config().enable_filename_constraint + } + + fn is_glob_pattern(&self, token: &str) -> bool { + GrepConfig.is_glob_pattern(token) + } +} diff --git a/crates/fff-python/Cargo.toml b/crates/fff-python/Cargo.toml new file mode 100644 index 0000000..8112f10 --- /dev/null +++ b/crates/fff-python/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fff-python" +version = "0.9.6" +edition = "2024" + +[lib] +name = "fff_python" +crate-type = ["cdylib"] + +[features] +# Pure-Rust walker by default; zlob is opt-in (needs Zig). +default = ["ripgrep"] +ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"] +zlob = ["fff/zlob", "fff-query-parser/zlob"] + +[dependencies] +fff = { package = "fff-search", path = "../fff-core", version = "0.9.6", default-features = false } +fff-query-parser = { path = "../fff-query-parser", version = "0.9.6", default-features = false } +git2 = { workspace = true } +pyo3 = { version = "0.24.0", features = ["extension-module", "abi3-py310"] } diff --git a/crates/fff-python/src/conversions.rs b/crates/fff-python/src/conversions.rs new file mode 100644 index 0000000..1656632 --- /dev/null +++ b/crates/fff-python/src/conversions.rs @@ -0,0 +1,107 @@ +use fff::file_picker::FilePicker; + +use crate::types::{DirItem, FileItem, GrepMatch, MatchRange, MixedDirItem, MixedFileItem, Score}; + +pub enum MixedItem { + File(MixedFileItem), + Dir(MixedDirItem), +} + +impl From<&fff::Score> for Score { + fn from(s: &fff::Score) -> Self { + Self { + total: s.total, + base_score: s.base_score, + filename_bonus: s.filename_bonus, + special_filename_bonus: s.special_filename_bonus, + frecency_boost: s.frecency_boost, + distance_penalty: s.distance_penalty, + current_file_penalty: s.current_file_penalty, + combo_match_boost: s.combo_match_boost, + path_alignment_bonus: s.path_alignment_bonus, + exact_match: s.exact_match, + match_type: s.match_type.to_string(), + } + } +} + +impl From<(&fff::FileItem, &FilePicker)> for FileItem { + fn from((item, picker): (&fff::FileItem, &FilePicker)) -> Self { + Self { + relative_path: item.relative_path(picker), + file_name: item.file_name(picker), + git_status: fff::git::format_git_status(item.git_status).to_string(), + size: item.size, + modified: item.modified, + access_frecency_score: item.access_frecency_score as i64, + modification_frecency_score: item.modification_frecency_score as i64, + total_frecency_score: item.total_frecency_score() as i64, + is_binary: item.is_binary(), + } + } +} + +impl From<(&fff::DirItem, &FilePicker)> for DirItem { + fn from((item, picker): (&fff::DirItem, &FilePicker)) -> Self { + Self { + relative_path: item.relative_path(picker), + dir_name: item.dir_name(picker), + max_access_frecency: item.max_access_frecency(), + } + } +} + +impl From<(&fff::FileItem, &FilePicker)> for MixedFileItem { + fn from((item, picker): (&fff::FileItem, &FilePicker)) -> Self { + Self { + relative_path: item.relative_path(picker), + file_name: item.file_name(picker), + git_status: fff::git::format_git_status(item.git_status).to_string(), + size: item.size, + modified: item.modified, + access_frecency_score: item.access_frecency_score as i64, + modification_frecency_score: item.modification_frecency_score as i64, + total_frecency_score: item.total_frecency_score() as i64, + is_binary: item.is_binary(), + } + } +} + +impl From<(&fff::DirItem, &FilePicker)> for MixedDirItem { + fn from((item, picker): (&fff::DirItem, &FilePicker)) -> Self { + Self { + relative_path: item.relative_path(picker), + dir_name: item.dir_name(picker), + max_access_frecency: item.max_access_frecency(), + } + } +} + +impl From<(&fff::GrepMatch, &fff::FileItem, &FilePicker)> for GrepMatch { + fn from((m, file, picker): (&fff::GrepMatch, &fff::FileItem, &FilePicker)) -> Self { + Self { + relative_path: file.relative_path(picker), + file_name: file.file_name(picker), + git_status: fff::git::format_git_status(file.git_status).to_string(), + line_content: m.line_content.clone(), + match_ranges: m + .match_byte_offsets + .iter() + .map(|&(s, e)| MatchRange { start: s, end: e }) + .collect(), + context_before: m.context_before.clone(), + context_after: m.context_after.clone(), + size: file.size, + modified: file.modified, + total_frecency_score: file.total_frecency_score() as i64, + access_frecency_score: file.access_frecency_score as i64, + modification_frecency_score: file.modification_frecency_score as i64, + line_number: m.line_number, + byte_offset: m.byte_offset, + col: m.col as u32, + fuzzy_score: m.fuzzy_score, + is_definition: m.is_definition, + is_binary: file.is_binary(), + } + } +} diff --git a/crates/fff-python/src/finder.rs b/crates/fff-python/src/finder.rs new file mode 100644 index 0000000..3384ae7 --- /dev/null +++ b/crates/fff-python/src/finder.rs @@ -0,0 +1,982 @@ +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use fff::file_picker::FilePicker; +use fff::frecency::FrecencyTracker; +use fff::query_tracker::QueryTracker; +use fff::{ + FFFMode, FilePickerOptions, FuzzySearchOptions, GrepSearchOptions, PaginationArgs, QueryParser, + SharedFilePicker, SharedFrecency, SharedQueryTracker, +}; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +use crate::conversions::MixedItem; +use crate::types::{ + DirItem, DirSearchResult, FileItem, GrepCursor, GrepMatch, GrepResult, MixedDirItem, + MixedFileItem, MixedSearchResult, ScanProgress, Score, SearchResult, +}; +use crate::{parse_grep_mode, py_err}; + +const DEFAULT_SEARCH_PAGE_SIZE: usize = 100; +// Sentinel-to-default conversion for combo boosting, mirroring the C/Node +// bindings: `0` means "use the engine default", not "disable". +const DEFAULT_COMBO_BOOST_MULTIPLIER: i32 = 100; +const DEFAULT_MIN_COMBO_COUNT: u32 = 3; + +fn defaulted_usize(value: u32, default: usize) -> usize { + if value == 0 { default } else { value as usize } +} + +fn defaulted_u64(value: u64, default: u64) -> u64 { + if value == 0 { default } else { value } +} + +fn defaulted_i32(value: i32, default: i32) -> i32 { + if value == 0 { default } else { value } +} + +fn defaulted_u32(value: u32, default: u32) -> u32 { + if value == 0 { default } else { value } +} + +fn create_parent_dir(path: &Path) -> PyResult<()> { + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent).map_err(py_err)?; + } + Ok(()) +} + +fn pagination_args(page_index: u32, page_size: u32) -> PaginationArgs { + PaginationArgs { + offset: page_index as usize, + limit: defaulted_usize(page_size, DEFAULT_SEARCH_PAGE_SIZE), + } +} + +fn fuzzy_options<'a>( + max_threads: u32, + current_file: Option<&'a str>, + project_path: &'a Path, + page_index: u32, + page_size: u32, + combo_boost_score_multiplier: i32, + min_combo_count: u32, +) -> FuzzySearchOptions<'a> { + FuzzySearchOptions { + max_threads: max_threads as usize, + current_file, + project_path: Some(project_path), + combo_boost_score_multiplier, + min_combo_count, + pagination: pagination_args(page_index, page_size), + } +} + +#[allow(clippy::too_many_arguments)] +fn grep_options( + mode: fff::GrepMode, + cursor_offset: usize, + max_file_size: u64, + max_matches_per_file: u32, + smart_case: bool, + page_limit: u32, + time_budget_ms: u64, + before_context: u32, + after_context: u32, + classify_definitions: bool, +) -> GrepSearchOptions { + let defaults = GrepSearchOptions::default(); + GrepSearchOptions { + max_file_size: defaulted_u64(max_file_size, defaults.max_file_size), + max_matches_per_file: max_matches_per_file as usize, + smart_case, + file_offset: cursor_offset, + page_limit: defaulted_usize(page_limit, defaults.page_limit), + mode, + time_budget_ms, + before_context: before_context as usize, + after_context: after_context as usize, + classify_definitions, + trim_whitespace: false, + abort_signal: None, + } +} + +fn convert_scores(scores: &[fff::Score]) -> Vec { + scores.iter().map(Score::from).collect() +} + +fn convert_grep_result(result: fff::grep::GrepResult<'_>, picker: &FilePicker) -> GrepResult { + let items = result + .matches + .iter() + .map(|m| GrepMatch::from((m, result.files[m.file_index], picker))) + .collect(); + + GrepResult { + items, + total_matched: result.matches.len() as u32, + total_files_searched: result.total_files_searched as u32, + total_files: result.total_files as u32, + filtered_file_count: result.filtered_file_count as u32, + next_file_offset: result.next_file_offset as u32, + regex_fallback_error: result.regex_fallback_error, + } +} + +fn clear_shared_state( + picker: &SharedFilePicker, + frecency: &SharedFrecency, + query_tracker: &SharedQueryTracker, +) { + if let Ok(mut guard) = picker.write() { + guard.take(); + } + if let Ok(mut guard) = frecency.write() { + *guard = None; + } + if let Ok(mut guard) = query_tracker.write() { + *guard = None; + } +} + +#[pyclass(subclass)] +pub struct FileFinder { + picker: SharedFilePicker, + frecency: SharedFrecency, + query_tracker: SharedQueryTracker, + cache_budget_max_files: usize, + cache_budget_max_bytes: u64, + cache_budget_max_file_size: u64, +} + +impl Drop for FileFinder { + fn drop(&mut self) { + clear_shared_state(&self.picker, &self.frecency, &self.query_tracker); + } +} + +#[pymethods] +impl FileFinder { + #[new] + #[pyo3(signature = ( + base_path, + *, + frecency_db_path=None, + history_db_path=None, + enable_mmap_cache=true, + enable_content_indexing=true, + watch=true, + ai_mode=false, + log_file_path=None, + log_level=None, + cache_budget_max_files=0, + cache_budget_max_bytes=0, + cache_budget_max_file_size=0, + enable_fs_root_scanning=false, + enable_home_dir_scanning=false, + follow_symlinks=false, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + py: Python<'_>, + base_path: PathBuf, + frecency_db_path: Option, + history_db_path: Option, + enable_mmap_cache: bool, + enable_content_indexing: bool, + watch: bool, + ai_mode: bool, + log_file_path: Option, + log_level: Option, + cache_budget_max_files: u64, + cache_budget_max_bytes: u64, + cache_budget_max_file_size: u64, + enable_fs_root_scanning: bool, + enable_home_dir_scanning: bool, + follow_symlinks: bool, + ) -> PyResult { + let shared_picker = SharedFilePicker::default(); + let shared_frecency = SharedFrecency::default(); + let query_tracker = SharedQueryTracker::default(); + let cache_budget_max_files = cache_budget_max_files as usize; + + let init_picker = shared_picker.clone(); + let init_frecency = shared_frecency.clone(); + let init_query_tracker = query_tracker.clone(); + + py.allow_threads(move || -> PyResult<()> { + if let Some(path) = frecency_db_path { + create_parent_dir(&path)?; + let tracker = FrecencyTracker::open(&path).map_err(py_err)?; + init_frecency.init(tracker).map_err(py_err)?; + } + + if let Some(path) = history_db_path { + create_parent_dir(&path)?; + let tracker = QueryTracker::open(&path).map_err(py_err)?; + init_query_tracker.init(tracker).map_err(py_err)?; + } + + if let Some(path) = log_file_path { + create_parent_dir(&path)?; + let path = path.to_string_lossy(); + fff::log::init_tracing(&path, log_level.as_deref(), None).map_err(py_err)?; + } + + let mode = if ai_mode { + FFFMode::Ai + } else { + FFFMode::Neovim + }; + + FilePicker::new_with_shared_state( + init_picker, + init_frecency, + FilePickerOptions { + base_path: base_path.to_string_lossy().to_string(), + enable_mmap_cache, + enable_content_indexing, + watch, + mode, + cache_budget: fff::ContentCacheBudget::from_overrides( + cache_budget_max_files, + cache_budget_max_bytes, + cache_budget_max_file_size, + ), + follow_symlinks, + enable_fs_root_scanning, + enable_home_dir_scanning, + }, + ) + .map_err(py_err) + })?; + + Ok(Self { + picker: shared_picker, + frecency: shared_frecency, + query_tracker, + cache_budget_max_files, + cache_budget_max_bytes, + cache_budget_max_file_size, + }) + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __exit__(&mut self, _exc_type: PyObject, _exc_value: PyObject, _traceback: PyObject) { + let _ = self.close(); + } + + fn close(&mut self) -> PyResult<()> { + clear_shared_state(&self.picker, &self.frecency, &self.query_tracker); + Ok(()) + } + + #[getter] + fn closed(&self) -> PyResult { + Ok(self.picker.read().map_err(py_err)?.is_none()) + } + + #[getter] + fn base_path(&self) -> PyResult> { + let guard = self.picker.read().map_err(py_err)?; + Ok(guard + .as_ref() + .map(|p| p.base_path().to_string_lossy().to_string())) + } + + #[getter] + fn scan_progress(&self) -> PyResult { + let guard = self.picker.read().map_err(py_err)?; + let picker = guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + let p = picker.get_scan_progress(); + Ok(ScanProgress { + scanned_files_count: p.scanned_files_count as u64, + is_scanning: p.is_scanning, + is_watcher_ready: p.is_watcher_ready, + is_warmup_complete: p.is_warmup_complete, + }) + } + + fn __repr__(&self) -> PyResult { + let guard = self.picker.read().map_err(py_err)?; + if let Some(ref picker) = *guard { + Ok(format!( + "FileFinder(base_path={:?}, closed=False)", + picker.base_path().to_string_lossy() + )) + } else { + Ok("FileFinder(closed=True)".to_string()) + } + } + + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + query, + *, + current_file=None, + max_threads=0, + page_index=0, + page_size=0, + combo_boost_score_multiplier=0, + min_combo_count=0, + ))] + fn search( + &self, + py: Python<'_>, + query: &str, + current_file: Option, + max_threads: u32, + page_index: u32, + page_size: u32, + combo_boost_score_multiplier: i32, + min_combo_count: u32, + ) -> PyResult { + let picker = self.picker.clone(); + let query_tracker = self.query_tracker.clone(); + let query = query.to_string(); + + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + let qt_guard = query_tracker.read().map_err(py_err)?; + + let parsed = QueryParser::default().parse(&query); + let result = picker.fuzzy_search( + &parsed, + qt_guard.as_ref(), + fuzzy_options( + max_threads, + current_file.as_deref(), + picker.base_path(), + page_index, + page_size, + defaulted_i32(combo_boost_score_multiplier, DEFAULT_COMBO_BOOST_MULTIPLIER), + defaulted_u32(min_combo_count, DEFAULT_MIN_COMBO_COUNT), + ), + ); + + Ok(SearchResult { + items: result + .items + .iter() + .map(|i| FileItem::from((*i, picker))) + .collect(), + scores: convert_scores(&result.scores), + total_matched: result.total_matched as u32, + total_files: result.total_files as u32, + }) + }) + } + + #[pyo3(signature = ( + pattern, + *, + current_file=None, + max_threads=0, + page_index=0, + page_size=0, + ))] + fn glob( + &self, + py: Python<'_>, + pattern: &str, + current_file: Option, + max_threads: u32, + page_index: u32, + page_size: u32, + ) -> PyResult { + let picker = self.picker.clone(); + let pattern = pattern.to_string(); + + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + + let result = picker.glob( + &pattern, + fuzzy_options( + max_threads, + current_file.as_deref(), + picker.base_path(), + page_index, + page_size, + 0, + 0, + ), + ); + + Ok(SearchResult { + items: result + .items + .iter() + .map(|i| FileItem::from((*i, picker))) + .collect(), + scores: convert_scores(&result.scores), + total_matched: result.total_matched as u32, + total_files: result.total_files as u32, + }) + }) + } + + #[pyo3(signature = ( + query, + *, + current_file=None, + max_threads=0, + page_index=0, + page_size=0, + ))] + fn directory_search( + &self, + py: Python<'_>, + query: &str, + current_file: Option, + max_threads: u32, + page_index: u32, + page_size: u32, + ) -> PyResult { + let picker = self.picker.clone(); + let query = query.to_string(); + + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + + let parsed = QueryParser::new(fff_query_parser::DirSearchConfig).parse(&query); + let result = picker.fuzzy_search_directories( + &parsed, + fuzzy_options( + max_threads, + current_file.as_deref(), + picker.base_path(), + page_index, + page_size, + 0, + 0, + ), + ); + + Ok(DirSearchResult { + items: result + .items + .iter() + .map(|i| DirItem::from((*i, picker))) + .collect(), + scores: convert_scores(&result.scores), + total_matched: result.total_matched as u32, + total_dirs: result.total_dirs as u32, + }) + }) + } + + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + query, + *, + current_file=None, + max_threads=0, + page_index=0, + page_size=0, + combo_boost_score_multiplier=0, + min_combo_count=0, + ))] + fn mixed_search( + &self, + py: Python<'_>, + query: &str, + current_file: Option, + max_threads: u32, + page_index: u32, + page_size: u32, + combo_boost_score_multiplier: i32, + min_combo_count: u32, + ) -> PyResult { + let picker = self.picker.clone(); + let query_tracker = self.query_tracker.clone(); + let query = query.to_string(); + + let (items, scores, total_matched, total_files, total_dirs) = + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + let qt_guard = query_tracker.read().map_err(py_err)?; + + let parsed = QueryParser::new(fff_query_parser::MixedSearchConfig).parse(&query); + let result = picker.fuzzy_search_mixed( + &parsed, + qt_guard.as_ref(), + fuzzy_options( + max_threads, + current_file.as_deref(), + picker.base_path(), + page_index, + page_size, + defaulted_i32(combo_boost_score_multiplier, DEFAULT_COMBO_BOOST_MULTIPLIER), + defaulted_u32(min_combo_count, DEFAULT_MIN_COMBO_COUNT), + ), + ); + + let items: Vec = result + .items + .iter() + .map(|item| match item { + fff::MixedItemRef::File(file) => { + MixedItem::File(MixedFileItem::from((*file, picker))) + } + fff::MixedItemRef::Dir(dir) => { + MixedItem::Dir(MixedDirItem::from((*dir, picker))) + } + }) + .collect(); + + Ok(( + items, + convert_scores(&result.scores), + result.total_matched as u32, + result.total_files as u32, + result.total_dirs as u32, + )) + })?; + + let items: PyResult> = items + .into_iter() + .map(|item| match item { + MixedItem::File(file) => Ok(Py::new(py, file)?.into_any()), + MixedItem::Dir(dir) => Ok(Py::new(py, dir)?.into_any()), + }) + .collect(); + + Ok(MixedSearchResult { + items: items?, + scores, + total_matched, + total_files, + total_dirs, + }) + } + + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + query, + *, + mode="plain", + max_file_size=0, + max_matches_per_file=0, + smart_case=true, + cursor=None, + page_limit=0, + time_budget_ms=0, + before_context=0, + after_context=0, + classify_definitions=false, + ))] + fn grep( + &self, + py: Python<'_>, + query: &str, + mode: &str, + max_file_size: u64, + max_matches_per_file: u32, + smart_case: bool, + cursor: Option<&GrepCursor>, + page_limit: u32, + time_budget_ms: u64, + before_context: u32, + after_context: u32, + classify_definitions: bool, + ) -> PyResult { + let picker = self.picker.clone(); + let query = query.to_string(); + let mode = parse_grep_mode(mode)?; + let cursor_offset = cursor.map(|c| c.offset as usize).unwrap_or(0); + + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + + let parsed = if picker.mode().is_ai() { + QueryParser::new(fff_query_parser::AiGrepConfig).parse(&query) + } else { + fff::grep::parse_grep_query(&query) + }; + let options = grep_options( + mode, + cursor_offset, + max_file_size, + max_matches_per_file, + smart_case, + page_limit, + time_budget_ms, + before_context, + after_context, + classify_definitions, + ); + + Ok(convert_grep_result(picker.grep(&parsed, &options), picker)) + }) + } + + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + patterns, + *, + constraints=None, + mode="plain", + max_file_size=0, + max_matches_per_file=0, + smart_case=true, + cursor=None, + page_limit=0, + time_budget_ms=0, + before_context=0, + after_context=0, + classify_definitions=false, + ))] + fn multi_grep( + &self, + py: Python<'_>, + patterns: Vec, + constraints: Option, + mode: &str, + max_file_size: u64, + max_matches_per_file: u32, + smart_case: bool, + cursor: Option<&GrepCursor>, + page_limit: u32, + time_budget_ms: u64, + before_context: u32, + after_context: u32, + classify_definitions: bool, + ) -> PyResult { + let picker = self.picker.clone(); + let mode = parse_grep_mode(mode)?; + let cursor_offset = cursor.map(|c| c.offset as usize).unwrap_or(0); + + py.allow_threads(move || -> PyResult<_> { + let picker_guard = picker.read().map_err(py_err)?; + let picker = picker_guard + .as_ref() + .ok_or_else(|| py_err("File picker not initialized"))?; + + if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) { + return Err(py_err("patterns must not be empty")); + } + let pattern_refs: Vec<&str> = patterns.iter().map(|s| s.as_str()).collect(); + + let parsed_constraints = constraints.as_ref().map(|c| { + if picker.mode().is_ai() { + QueryParser::new(fff_query_parser::AiGrepConfig).parse(c) + } else { + fff::grep::parse_grep_query(c) + } + }); + let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints { + Some(q) => &q.constraints, + None => &[], + }; + let options = grep_options( + mode, + cursor_offset, + max_file_size, + max_matches_per_file, + smart_case, + page_limit, + time_budget_ms, + before_context, + after_context, + classify_definitions, + ); + + Ok(convert_grep_result( + picker.multi_grep(&pattern_refs, constraint_refs, &options), + picker, + )) + }) + } + + fn scan_files(&self) -> PyResult<()> { + self.picker + .trigger_full_rescan_async(&self.frecency) + .map_err(py_err) + } + + fn is_scanning(&self) -> PyResult { + let guard = self.picker.read().map_err(py_err)?; + Ok(guard.as_ref().map(|p| p.is_scan_active()).unwrap_or(false)) + } + + #[pyo3(signature = (timeout_ms=5000))] + fn wait_for_scan_blocking(&self, py: Python<'_>, timeout_ms: u64) -> PyResult { + let picker = self.picker.clone(); + py.allow_threads(move || Ok(picker.wait_for_scan(Duration::from_millis(timeout_ms)))) + } + + fn reindex(&self, py: Python<'_>, new_path: PathBuf) -> PyResult<()> { + let picker = self.picker.clone(); + let frecency = self.frecency.clone(); + let cache_budget_max_files = self.cache_budget_max_files; + let cache_budget_max_bytes = self.cache_budget_max_bytes; + let cache_budget_max_file_size = self.cache_budget_max_file_size; + + py.allow_threads(move || -> PyResult<()> { + if !new_path.exists() { + return Err(py_err(format!( + "Path does not exist: {}", + new_path.display() + ))); + } + let canonical = fff::path_utils::canonicalize(&new_path).map_err(py_err)?; + + let (warmup_caches, content_indexing, watch, mode, fs_root, home_dir, follow_symlinks) = { + let guard = picker.read().map_err(py_err)?; + if let Some(ref picker) = *guard { + ( + picker.has_mmap_cache(), + picker.has_content_indexing(), + picker.has_watcher(), + picker.mode(), + picker.fs_root_scanning_enabled(), + picker.home_dir_scanning_enabled(), + picker.follows_symlinks(), + ) + } else { + (false, true, true, FFFMode::default(), false, false, false) + } + }; + + FilePicker::new_with_shared_state( + picker.clone(), + frecency, + FilePickerOptions { + base_path: canonical.to_string_lossy().to_string(), + enable_mmap_cache: warmup_caches, + enable_content_indexing: content_indexing, + watch, + mode, + cache_budget: fff::ContentCacheBudget::from_overrides( + cache_budget_max_files, + cache_budget_max_bytes, + cache_budget_max_file_size, + ), + follow_symlinks, + enable_fs_root_scanning: fs_root, + enable_home_dir_scanning: home_dir, + }, + ) + .map_err(py_err) + }) + } + + fn refresh_git_status(&self, py: Python<'_>) -> PyResult { + let picker = self.picker.clone(); + let frecency = self.frecency.clone(); + py.allow_threads(move || { + picker + .refresh_git_status(&frecency) + .map_err(py_err) + .map(|c| c as i64) + }) + } + + #[pyo3(signature = (query, selected_file_path))] + fn track_query( + &self, + py: Python<'_>, + query: &str, + selected_file_path: PathBuf, + ) -> PyResult { + let picker = self.picker.clone(); + let query_tracker = self.query_tracker.clone(); + let query = query.to_string(); + + py.allow_threads(move || -> PyResult { + let file_path = fff::path_utils::canonicalize(&selected_file_path).map_err(py_err)?; + let project_path = { + let guard = picker.read().map_err(py_err)?; + guard.as_ref().map(|p| p.base_path().to_path_buf()) + }; + let project_path = match project_path { + Some(p) => p, + None => return Ok(false), + }; + + let mut qt_guard = query_tracker.write().map_err(py_err)?; + if let Some(ref mut tracker) = *qt_guard { + tracker + .track_query_completion(&query, &project_path, &file_path) + .map_err(py_err)?; + Ok(true) + } else { + Ok(false) + } + }) + } + + fn get_historical_query(&self, py: Python<'_>, offset: u64) -> PyResult> { + let picker = self.picker.clone(); + let query_tracker = self.query_tracker.clone(); + + py.allow_threads(move || -> PyResult> { + let project_path = { + let guard = picker.read().map_err(py_err)?; + guard.as_ref().map(|p| p.base_path().to_path_buf()) + }; + let project_path = match project_path { + Some(p) => p, + None => return Ok(None), + }; + + let qt_guard = query_tracker.read().map_err(py_err)?; + if let Some(ref tracker) = *qt_guard { + tracker + .get_historical_query(&project_path, offset as usize) + .map_err(py_err) + } else { + Ok(None) + } + }) + } + + #[pyo3(signature = (test_path=None))] + fn health_check(&self, py: Python<'_>, test_path: Option) -> PyResult> { + let picker = self.picker.clone(); + let frecency = self.frecency.clone(); + let query_tracker = self.query_tracker.clone(); + + let ( + git_version, + repository_found, + workdir, + git_error, + picker_initialized, + picker_base_path, + picker_is_scanning, + picker_indexed_files, + frecency_initialized, + query_tracker_initialized, + ) = py.allow_threads(move || -> PyResult<_> { + // Resolve the path to inspect: explicit arg → indexed base path → + // process cwd. Report a cwd-resolution failure instead of silently + // discovering from an empty path. + let (test_path, cwd_error) = match test_path { + Some(p) => (Some(p), None), + None => { + let base = picker + .read() + .ok() + .and_then(|g| g.as_ref().map(|p| p.base_path().to_path_buf())); + match base { + Some(p) => (Some(p), None), + None => match std::env::current_dir() { + Ok(p) => (Some(p), None), + Err(e) => ( + None, + Some(format!("could not determine current directory: {}", e)), + ), + }, + } + } + }; + + let git_version = git2::Version::get(); + let (major, minor, rev) = git_version.libgit2_version(); + let git_version = format!("{}.{}.{}", major, minor, rev); + let (repository_found, workdir, git_error) = match test_path { + None => (false, None, cwd_error), + Some(test_path) => match git2::Repository::discover(&test_path) { + Ok(repo) => ( + true, + repo.workdir().map(|p| p.to_string_lossy().to_string()), + None, + ), + Err(e) => (false, None, Some(e.message().to_string())), + }, + }; + + let (picker_initialized, picker_base_path, picker_is_scanning, picker_indexed_files) = { + let guard = picker.read().map_err(py_err)?; + if let Some(ref picker) = *guard { + let progress = picker.get_scan_progress(); + ( + true, + Some(picker.base_path().to_string_lossy().to_string()), + Some(picker.is_scan_active()), + Some(progress.scanned_files_count), + ) + } else { + (false, None, None, None) + } + }; + let frecency_initialized = frecency.read().map_err(py_err)?.is_some(); + let query_tracker_initialized = query_tracker.read().map_err(py_err)?.is_some(); + + Ok(( + git_version, + repository_found, + workdir, + git_error, + picker_initialized, + picker_base_path, + picker_is_scanning, + picker_indexed_files, + frecency_initialized, + query_tracker_initialized, + )) + })?; + + let dict = PyDict::new(py); + dict.set_item("version", env!("CARGO_PKG_VERSION"))?; + + let git_info = PyDict::new(py); + git_info.set_item("available", true)?; + git_info.set_item("libgit2_version", git_version)?; + git_info.set_item("repository_found", repository_found)?; + if let Some(workdir) = workdir { + git_info.set_item("workdir", workdir)?; + } + if let Some(error) = git_error { + git_info.set_item("error", error)?; + } + dict.set_item("git", git_info)?; + + let picker_info = PyDict::new(py); + picker_info.set_item("initialized", picker_initialized)?; + if let Some(base_path) = picker_base_path { + picker_info.set_item("base_path", base_path)?; + } + if let Some(is_scanning) = picker_is_scanning { + picker_info.set_item("is_scanning", is_scanning)?; + } + if let Some(indexed_files) = picker_indexed_files { + picker_info.set_item("indexed_files", indexed_files)?; + } + dict.set_item("file_picker", picker_info)?; + + let frecency_info = PyDict::new(py); + frecency_info.set_item("initialized", frecency_initialized)?; + dict.set_item("frecency", frecency_info)?; + + let query_info = PyDict::new(py); + query_info.set_item("initialized", query_tracker_initialized)?; + dict.set_item("query_tracker", query_info)?; + + Ok(dict.unbind()) + } +} diff --git a/crates/fff-python/src/lib.rs b/crates/fff-python/src/lib.rs new file mode 100644 index 0000000..94d2194 --- /dev/null +++ b/crates/fff-python/src/lib.rs @@ -0,0 +1,44 @@ +use pyo3::create_exception; +use pyo3::prelude::*; + +mod conversions; +mod finder; +mod types; + +create_exception!(fff_python, FFFException, pyo3::exceptions::PyException); + +fn py_err(e: E) -> PyErr { + PyErr::new::(format!("{}", e)) +} + +pub fn parse_grep_mode(mode: &str) -> PyResult { + match mode { + "plain" => Ok(fff::GrepMode::PlainText), + "regex" => Ok(fff::GrepMode::Regex), + "fuzzy" => Ok(fff::GrepMode::Fuzzy), + _ => Err(py_err(format!( + "invalid grep mode: {:?}. Must be one of: plain, regex, fuzzy", + mode + ))), + } +} + +#[pymodule] +fn _fff_python(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add("FFFException", m.py().get_type::())?; + Ok(()) +} diff --git a/crates/fff-python/src/types.rs b/crates/fff-python/src/types.rs new file mode 100644 index 0000000..32c2f13 --- /dev/null +++ b/crates/fff-python/src/types.rs @@ -0,0 +1,408 @@ +use pyo3::prelude::*; + +#[pyclass] +#[derive(Clone)] +pub struct Score { + #[pyo3(get)] + pub total: i32, + #[pyo3(get)] + pub base_score: i32, + #[pyo3(get)] + pub filename_bonus: i32, + #[pyo3(get)] + pub special_filename_bonus: i32, + #[pyo3(get)] + pub frecency_boost: i32, + #[pyo3(get)] + pub distance_penalty: i32, + #[pyo3(get)] + pub current_file_penalty: i32, + #[pyo3(get)] + pub combo_match_boost: i32, + #[pyo3(get)] + pub path_alignment_bonus: i32, + #[pyo3(get)] + pub exact_match: bool, + #[pyo3(get)] + pub match_type: String, +} + +#[pymethods] +impl Score { + fn __repr__(&self) -> String { + format!( + "Score(total={}, base_score={}, filename_bonus={}, match_type={:?})", + self.total, self.base_score, self.filename_bonus, self.match_type + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct FileItem { + #[pyo3(get)] + pub relative_path: String, + #[pyo3(get)] + pub file_name: String, + #[pyo3(get)] + pub git_status: String, + #[pyo3(get)] + pub size: u64, + #[pyo3(get)] + pub modified: u64, + #[pyo3(get)] + pub access_frecency_score: i64, + #[pyo3(get)] + pub modification_frecency_score: i64, + #[pyo3(get)] + pub total_frecency_score: i64, + #[pyo3(get)] + pub is_binary: bool, +} + +#[pymethods] +impl FileItem { + fn __repr__(&self) -> String { + format!( + "FileItem(relative_path={:?}, file_name={:?}, size={})", + self.relative_path, self.file_name, self.size + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct DirItem { + #[pyo3(get)] + pub relative_path: String, + #[pyo3(get)] + pub dir_name: String, + #[pyo3(get)] + pub max_access_frecency: i32, +} + +#[pymethods] +impl DirItem { + fn __repr__(&self) -> String { + format!( + "DirItem(relative_path={:?}, dir_name={:?})", + self.relative_path, self.dir_name + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct MixedFileItem { + #[pyo3(get)] + pub relative_path: String, + #[pyo3(get)] + pub file_name: String, + #[pyo3(get)] + pub git_status: String, + #[pyo3(get)] + pub size: u64, + #[pyo3(get)] + pub modified: u64, + #[pyo3(get)] + pub access_frecency_score: i64, + #[pyo3(get)] + pub modification_frecency_score: i64, + #[pyo3(get)] + pub total_frecency_score: i64, + #[pyo3(get)] + pub is_binary: bool, +} + +#[pymethods] +impl MixedFileItem { + fn __repr__(&self) -> String { + format!( + "MixedFileItem(relative_path={:?}, file_name={:?}, size={})", + self.relative_path, self.file_name, self.size + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct MixedDirItem { + #[pyo3(get)] + pub relative_path: String, + #[pyo3(get)] + pub dir_name: String, + #[pyo3(get)] + pub max_access_frecency: i32, +} + +#[pymethods] +impl MixedDirItem { + fn __repr__(&self) -> String { + format!( + "MixedDirItem(relative_path={:?}, dir_name={:?})", + self.relative_path, self.dir_name + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct MatchRange { + #[pyo3(get)] + pub start: u32, + #[pyo3(get)] + pub end: u32, +} + +#[pymethods] +impl MatchRange { + fn __repr__(&self) -> String { + format!("MatchRange(start={}, end={})", self.start, self.end) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct GrepMatch { + #[pyo3(get)] + pub relative_path: String, + #[pyo3(get)] + pub file_name: String, + #[pyo3(get)] + pub git_status: String, + #[pyo3(get)] + pub line_content: String, + #[pyo3(get)] + pub match_ranges: Vec, + #[pyo3(get)] + pub context_before: Vec, + #[pyo3(get)] + pub context_after: Vec, + #[pyo3(get)] + pub size: u64, + #[pyo3(get)] + pub modified: u64, + #[pyo3(get)] + pub total_frecency_score: i64, + #[pyo3(get)] + pub access_frecency_score: i64, + #[pyo3(get)] + pub modification_frecency_score: i64, + #[pyo3(get)] + pub line_number: u64, + #[pyo3(get)] + pub byte_offset: u64, + #[pyo3(get)] + pub col: u32, + #[pyo3(get)] + pub fuzzy_score: Option, + #[pyo3(get)] + pub is_definition: bool, + #[pyo3(get)] + pub is_binary: bool, +} + +#[pymethods] +impl GrepMatch { + fn __repr__(&self) -> String { + format!( + "GrepMatch(relative_path={:?}, line_number={}, line_content={:?})", + self.relative_path, self.line_number, self.line_content + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct SearchResult { + #[pyo3(get)] + pub items: Vec, + #[pyo3(get)] + pub scores: Vec, + #[pyo3(get)] + pub total_matched: u32, + #[pyo3(get)] + pub total_files: u32, +} + +#[pymethods] +impl SearchResult { + fn __repr__(&self) -> String { + format!( + "SearchResult(items={}, total_matched={}, total_files={})", + self.items.len(), + self.total_matched, + self.total_files + ) + } + + fn __len__(&self) -> usize { + self.items.len() + } + + fn __bool__(&self) -> bool { + !self.items.is_empty() + } +} + +#[pyclass] +#[derive(Clone)] +pub struct DirSearchResult { + #[pyo3(get)] + pub items: Vec, + #[pyo3(get)] + pub scores: Vec, + #[pyo3(get)] + pub total_matched: u32, + #[pyo3(get)] + pub total_dirs: u32, +} + +#[pymethods] +impl DirSearchResult { + fn __repr__(&self) -> String { + format!( + "DirSearchResult(items={}, total_matched={}, total_dirs={})", + self.items.len(), + self.total_matched, + self.total_dirs + ) + } + + fn __len__(&self) -> usize { + self.items.len() + } + + fn __bool__(&self) -> bool { + !self.items.is_empty() + } +} + +#[pyclass] +pub struct MixedSearchResult { + #[pyo3(get)] + pub items: Vec, + #[pyo3(get)] + pub scores: Vec, + #[pyo3(get)] + pub total_matched: u32, + #[pyo3(get)] + pub total_files: u32, + #[pyo3(get)] + pub total_dirs: u32, +} + +#[pymethods] +impl MixedSearchResult { + fn __repr__(&self) -> String { + format!( + "MixedSearchResult(items={}, total_matched={}, total_files={}, total_dirs={})", + self.items.len(), + self.total_matched, + self.total_files, + self.total_dirs + ) + } + + fn __len__(&self) -> usize { + self.items.len() + } + + fn __bool__(&self) -> bool { + !self.items.is_empty() + } +} + +#[pyclass] +#[derive(Clone)] +pub struct GrepResult { + #[pyo3(get)] + pub items: Vec, + #[pyo3(get)] + pub total_matched: u32, + #[pyo3(get)] + pub total_files_searched: u32, + #[pyo3(get)] + pub total_files: u32, + #[pyo3(get)] + pub filtered_file_count: u32, + #[pyo3(get)] + pub next_file_offset: u32, + #[pyo3(get)] + pub regex_fallback_error: Option, +} + +#[pymethods] +impl GrepResult { + fn __repr__(&self) -> String { + format!( + "GrepResult(items={}, total_matched={}, next_file_offset={})", + self.items.len(), + self.total_matched, + self.next_file_offset + ) + } + + fn __len__(&self) -> usize { + self.items.len() + } + + fn __bool__(&self) -> bool { + !self.items.is_empty() + } + + #[getter] + fn has_more(&self) -> bool { + self.next_file_offset > 0 + } + + fn next_cursor(&self, py: Python<'_>) -> PyResult>> { + if self.next_file_offset > 0 { + Ok(Some(Py::new(py, GrepCursor::new(self.next_file_offset))?)) + } else { + Ok(None) + } + } +} + +#[pyclass] +#[derive(Clone)] +pub struct ScanProgress { + #[pyo3(get)] + pub scanned_files_count: u64, + #[pyo3(get)] + pub is_scanning: bool, + #[pyo3(get)] + pub is_watcher_ready: bool, + #[pyo3(get)] + pub is_warmup_complete: bool, +} + +#[pymethods] +impl ScanProgress { + fn __repr__(&self) -> String { + format!( + "ScanProgress(scanned_files_count={}, is_scanning={})", + self.scanned_files_count, self.is_scanning + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct GrepCursor { + #[pyo3(get)] + pub offset: u32, +} + +#[pymethods] +impl GrepCursor { + #[new] + fn new(offset: u32) -> Self { + Self { offset } + } + + fn __repr__(&self) -> String { + format!("GrepCursor(offset={})", self.offset) + } +} diff --git a/crates/fff-query-parser/Cargo.toml b/crates/fff-query-parser/Cargo.toml new file mode 100644 index 0000000..416d2db --- /dev/null +++ b/crates/fff-query-parser/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "fff-query-parser" +version = "0.9.6" +edition = "2024" +description = "Query parser for fff file finder - includes specific syntax for various constraints like globs, extensions, regex etc" +license = "MIT" +authors = ["Dmitriy Kovalenko "] + +[lib] +path = "src/lib.rs" + +[features] +# `ripgrep` = pure-Rust glob detection (default). `zlob` overrides it when set. +default = ["ripgrep"] +ripgrep = [] +zlob = ["dep:zlob"] + +[dependencies] +zlob = { workspace = true, optional = true } + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } + +[[bench]] +name = "parse_bench" +harness = false diff --git a/crates/fff-query-parser/benches/parse_bench.rs b/crates/fff-query-parser/benches/parse_bench.rs new file mode 100644 index 0000000..cd6e8d2 --- /dev/null +++ b/crates/fff-query-parser/benches/parse_bench.rs @@ -0,0 +1,180 @@ +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use fff_query_parser::*; + +fn bench_parse_simple(c: &mut Criterion) { + let parser = QueryParser::default(); + + c.bench_function("parse_simple_text", |b| { + b.iter(|| parser.parse(black_box("hello world"))); + }); + + c.bench_function("parse_extension", |b| { + b.iter(|| parser.parse(black_box("*.rs"))); + }); + + c.bench_function("parse_text_with_extension", |b| { + b.iter(|| parser.parse(black_box("name *.rs"))); + }); +} + +fn bench_parse_complex(c: &mut Criterion) { + let parser = QueryParser::default(); + + c.bench_function("parse_complex_mixed", |b| { + b.iter(|| parser.parse(black_box("src name *.rs !test /lib/ status:modified"))); + }); + + c.bench_function("parse_glob", |b| { + b.iter(|| parser.parse(black_box("**/*.rs"))); + }); + + c.bench_function("parse_multiple_constraints", |b| { + b.iter(|| parser.parse(black_box("*.rs *.toml *.md !test !node_modules /src/"))); + }); +} + +fn bench_parse_realistic_queries(c: &mut Criterion) { + let parser = QueryParser::default(); + + let queries = vec![ + "file", + "test", + "mod.rs", + "src/*.rs", + "lib test", + "*.rs !test", + "src/lib/*.rs", + "/src/ name", + "status:modified *.rs", + "type:rust test !node_modules", + ]; + + let mut group = c.benchmark_group("realistic_queries"); + for query in queries.iter() { + group.throughput(Throughput::Bytes(query.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(query), query, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + } + group.finish(); +} + +fn bench_parse_various_lengths(c: &mut Criterion) { + let parser = QueryParser::default(); + + let short = "*.rs"; + let medium = "src name *.rs !test"; + let long = "src lib test name *.rs *.toml !node_modules !test /src/ /lib/ status:modified"; + let very_long = + "a b c d e f g h i j k l m n o p q r s t u v w x y z *.rs *.toml *.md *.txt *.js"; + + let mut group = c.benchmark_group("query_lengths"); + + group.throughput(Throughput::Bytes(short.len() as u64)); + group.bench_with_input(BenchmarkId::new("short", short.len()), &short, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(medium.len() as u64)); + group.bench_with_input(BenchmarkId::new("medium", medium.len()), &medium, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(long.len() as u64)); + group.bench_with_input(BenchmarkId::new("long", long.len()), &long, |b, q| { + b.iter(|| parser.parse(black_box(q))); + }); + + group.throughput(Throughput::Bytes(very_long.len() as u64)); + group.bench_with_input( + BenchmarkId::new("very_long", very_long.len()), + &very_long, + |b, q| { + b.iter(|| parser.parse(black_box(q))); + }, + ); + + group.finish(); +} + +fn bench_config_comparison(c: &mut Criterion) { + let file_picker = QueryParser::new(FileSearchConfig); + let grep = QueryParser::new(GrepConfig); + + let query = "src name *.rs !test"; + + let mut group = c.benchmark_group("config_comparison"); + + group.bench_function("file_picker_config", |b| { + b.iter(|| file_picker.parse(black_box(query))); + }); + + group.bench_function("grep_config", |b| { + b.iter(|| grep.parse(black_box(query))); + }); + + group.finish(); +} + +fn bench_constraint_types(c: &mut Criterion) { + let parser = QueryParser::default(); + + let mut group = c.benchmark_group("constraint_types"); + + group.bench_function("extension", |b| { + b.iter(|| parser.parse(black_box("*.rs"))); + }); + + group.bench_function("glob", |b| { + b.iter(|| parser.parse(black_box("**/*.rs"))); + }); + + group.bench_function("exclude", |b| { + b.iter(|| parser.parse(black_box("!test"))); + }); + + group.bench_function("path_segment", |b| { + b.iter(|| parser.parse(black_box("/src/"))); + }); + + group.bench_function("git_status", |b| { + b.iter(|| parser.parse(black_box("status:modified"))); + }); + + group.bench_function("file_type", |b| { + b.iter(|| parser.parse(black_box("type:rust"))); + }); + + group.finish(); +} + +fn bench_worst_case(c: &mut Criterion) { + let parser = QueryParser::default(); + + // Worst case: many constraints that all need to be checked + let worst_case = "a b c d e f g h i j k l m n o p q r s t u v w x y z"; + + c.bench_function("worst_case_many_text_tokens", |b| { + b.iter(|| parser.parse(black_box(worst_case))); + }); + + // Many constraints + let many_constraints = "*.rs *.toml *.md *.txt *.js *.ts *.jsx *.tsx *.vue *.svelte"; + + c.bench_function("worst_case_many_constraints", |b| { + b.iter(|| parser.parse(black_box(many_constraints))); + }); +} + +criterion_group!( + benches, + bench_parse_simple, + bench_parse_complex, + bench_parse_realistic_queries, + bench_parse_various_lengths, + bench_config_comparison, + bench_constraint_types, + bench_worst_case, +); + +criterion_main!(benches); diff --git a/crates/fff-query-parser/src/config.rs b/crates/fff-query-parser/src/config.rs new file mode 100644 index 0000000..9516c96 --- /dev/null +++ b/crates/fff-query-parser/src/config.rs @@ -0,0 +1,248 @@ +use crate::constraints::Constraint; +use crate::glob_detect::has_wildcards; + +/// Check if a token looks like a filename/path for use as a `FilePath` constraint. +#[deprecated(note = "use `Constraint::is_filename_constraint_token`")] +pub fn is_filename_constraint_token(token: &str) -> bool { + Constraint::is_filename_constraint_token(token) +} + +/// Parser configuration trait - allows different picker types to customize parsing +pub trait ParserConfig { + fn enable_glob(&self) -> bool { + true + } + + /// Should parse extension shortcuts (e.g., *.rs) + fn enable_extension(&self) -> bool { + true + } + + /// Should parse exclusion patterns (e.g., !test) + fn enable_exclude(&self) -> bool { + true + } + + /// Should parse path segments (e.g., /src/) + fn enable_path_segments(&self) -> bool { + true + } + + /// Should parse type constraints (e.g., type:rust) + fn enable_type_filter(&self) -> bool { + true + } + + /// Should parse git status (e.g., status:modified) + fn enable_git_status(&self) -> bool { + true + } + + /// Should parse location suffixes (e.g., file:12, file:12:4) + /// Disabled for grep modes where colon-number patterns like localhost:8080 + /// are search text, not file locations. + fn enable_location(&self) -> bool { + true + } + + /// Determine whether a token should be treated as a glob constraint. + /// + /// The default implementation delegates to `zlob::has_wildcards` with + /// `RECOMMENDED` flags, which recognises `*`, `?`, `[`, `{…}` etc. + /// + /// Override this in configs where some wildcard characters are common + /// in search text (e.g. grep mode where `?` and `[` appear in code). + fn is_glob_pattern(&self, token: &str) -> bool { + has_wildcards(token) + } + + /// If `true`, a PathSegment constraint that is the ONLY token in the + /// query is demoted to fuzzy text to avoid over filtering + fn treat_lone_path_as_text(&self) -> bool { + true + } + + /// If `true`, tokens that look like filenames (`score.rs`, `src/main.rs`) + /// become `FilePath` constraints that scope the search to matching paths. + /// Off by default: a partial filename like `vite.conf` would otherwise + /// filter out `vite.config.ts` and yield zero results. + fn enable_filename_constraint(&self) -> bool { + false + } + + /// Custom constraint parsers for picker-specific needs. + /// + /// Returns `None` by default. Filename-token detection is handled + /// separately by the parser via `enable_filename_constraint`. + fn parse_custom<'a>(&self, _input: &'a str) -> Option> { + None + } +} + +/// Default configuration for the file picker. +/// +/// Filename-constraint detection is off (trait default), so partial filenames +/// like `vite.conf` don't silently filter out fuzzy matches. The Neovim layer +/// overrides `enable_filename_constraint` to opt in based on user config. +#[derive(Debug, Clone, Copy, Default)] +pub struct FileSearchConfig; + +impl ParserConfig for FileSearchConfig {} + +/// Configuration for full-text search (grep) - file constraints enabled for +/// filtering which files to search, git status disabled since it's not useful +/// when searching file contents. +/// +/// Glob detection is narrowed: only patterns containing a path separator (`/`) +/// or brace expansion (`{…}`) are treated as globs. Characters like `?` and +/// `[` are extremely common in source code and must remain literal search text. +#[derive(Debug, Clone, Copy, Default)] +pub struct GrepConfig; + +impl ParserConfig for GrepConfig { + fn enable_path_segments(&self) -> bool { + true + } + + fn enable_git_status(&self) -> bool { + false + } + + fn enable_location(&self) -> bool { + false + } + + /// Only recognise globs that are clearly directory/path oriented. + /// + /// Characters like `?`, `[`, and bare `*` (without `/`) are extremely + /// common in source code (`foo?`, `arr[0]`, `*ptr`) and must NOT be + /// consumed as glob constraints. We only treat a token as a glob when + /// it contains path-oriented patterns: + /// + /// - Contains `/` → path glob (e.g. `src/**/*.rs`, `*/tests/*`) + /// - Contains `{…}` → brace expansion (e.g. `{src,lib}`) + fn is_glob_pattern(&self, token: &str) -> bool { + // Must contain at least one glob wildcard character + if !has_wildcards(token) { + return false; + } + + let bytes = token.as_bytes(); + + // Contains path separator → clearly a path glob + if bytes.contains(&b'/') { + return true; + } + + // Brace expansion → useful for directory alternatives. + // Require a comma between `{` and `}` AND at least one letter to + // distinguish real glob expansions like `{src,lib}` or `*.{ts,tsx}` + // from code patterns like `format!("{}")` and regex quantifiers `{2,3}`. + // Guard `open < close` so reversed braces like `}{` don't panic on slicing. + if let Some(open) = bytes.iter().position(|&b| b == b'{') + && let Some(close) = bytes.iter().rposition(|&b| b == b'}') + && open < close + { + let inner = &bytes[open + 1..close]; + if inner.contains(&b',') && inner.iter().any(|b| b.is_ascii_alphabetic()) { + return true; + } + } + + // Everything else (?, [, bare * without /) → treat as literal text + false + } +} + +/// Configuration for AI-mode grep — extends `GrepConfig` behavior with +/// automatic file-path constraint detection. +/// +/// Bare filenames with valid extensions (`schema.rs`) and path-prefixed +/// filenames (`libswscale/input.c`) are detected as `FilePath` constraints +/// so the search is scoped to matching files. The caller validates the +/// constraint against the index and drops it if no files match (fallback). +#[derive(Debug, Clone, Copy, Default)] +pub struct AiGrepConfig; + +/// Configuration for directory and mixed search modes. +/// +/// Disables path segment parsing so that trailing `/` is kept as fuzzy text +/// (e.g. `fff-core/` fuzzy-matches directory paths instead of becoming a +/// `PathSegment("fff-core")` constraint with an empty query). Extension and +/// filename constraints are also disabled since they don't apply to directories. +#[derive(Debug, Clone, Copy, Default)] +pub struct DirSearchConfig; + +impl ParserConfig for AiGrepConfig { + fn enable_path_segments(&self) -> bool { + true + } + + fn enable_git_status(&self) -> bool { + false + } + + fn enable_location(&self) -> bool { + false + } + + fn enable_filename_constraint(&self) -> bool { + true + } + + fn is_glob_pattern(&self, token: &str) -> bool { + // First check GrepConfig's strict rules (path globs, brace expansion) + if GrepConfig.is_glob_pattern(token) { + return true; + } + + // AI agents use `*text*` to scope file searches (e.g. `*quote* TODO`). + // Recognise tokens that start AND end with `*` with non-empty text + // between them as glob constraints. Bare `*` or `**` are excluded. + if !has_wildcards(token) { + return false; + } + let bytes = token.as_bytes(); + if bytes.len() >= 3 + && bytes[0] == b'*' + && bytes[bytes.len() - 1] == b'*' + && bytes[1..bytes.len() - 1].iter().all(|&b| b != b'*') + { + return true; + } + + false + } +} + +impl ParserConfig for DirSearchConfig { + fn enable_path_segments(&self) -> bool { + false + } + + fn enable_extension(&self) -> bool { + false + } + + fn enable_type_filter(&self) -> bool { + false + } + + fn enable_git_status(&self) -> bool { + false + } +} + +/// Configuration for mixed (files + directories) search. +/// +/// Like `DirSearchConfig`, disables path segment parsing so trailing `/` +/// triggers dirs-only mode instead of becoming a constraint. Keeps git +/// status and extension filters enabled since files are part of the results. +#[derive(Debug, Clone, Copy, Default)] +pub struct MixedSearchConfig; + +impl ParserConfig for MixedSearchConfig { + fn enable_path_segments(&self) -> bool { + false + } +} diff --git a/crates/fff-query-parser/src/constraints.rs b/crates/fff-query-parser/src/constraints.rs new file mode 100644 index 0000000..74d2c68 --- /dev/null +++ b/crates/fff-query-parser/src/constraints.rs @@ -0,0 +1,83 @@ +use crate::glob_detect::has_wildcards; + +/// Constraint types that can be extracted from a query +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Constraint<'a> { + /// Match file extension: *.rs -> Extension("rs") + Extension(&'a str), + + /// Glob pattern: **/*.rs -> Glob("**/*.rs") + Glob(&'a str), + + /// Multiple text search parts: ["src", "name"] + /// Uses slice to avoid allocation + Parts(&'a [&'a str]), + + /// Single text token (optimized case) + Text(&'a str), + + /// Exclude pattern: !test -> Exclude(&["test"]) + Exclude(&'a [&'a str]), + + /// Path constraint: /src/ -> PathSegment("src") + PathSegment(&'a str), + + /// File path constraint (AI mode): "libswscale/input.c" → FilePath("libswscale/input.c") + /// Matches files whose relative path ends with this suffix at a `/` boundary. + FilePath(&'a str), + + /// File type constraint: type:rust -> FileType("rust") + FileType(&'a str), + + /// Git status constraint: status:modified -> GitStatus(Modified) + GitStatus(GitStatusFilter), + + /// Negation constraint: !extension:rs -> Not(Extension("rs")) + /// Negates the inner constraint + Not(Box>), +} + +impl Constraint<'_> { + #[inline(always)] + pub fn is_filename_constraint_token(token: &str) -> bool { + let bytes = token.as_bytes(); + + // Must NOT end with / or . + if token.is_empty() || (bytes.last() == Some(&b'/') && bytes.first() != Some(&b'.')) { + return false; + } + + // Must NOT contain wildcards (those are globs) + if has_wildcards(token) { + return false; + } + + // Get the filename component (after last /) + let filename = token.rsplit('/').next().unwrap_or(token); + + // Extension must exist and look like a real file extension: + // starts with an ASCII letter (rejects version numbers like "v2.0"), + // followed by alphanumeric chars, max 10 chars total. + match filename.rfind('.') { + None => false, + Some(dot_idx) => { + let extension = &filename[dot_idx + 1..]; + + !extension.is_empty() + && extension.len() <= 10 // just an sassumption + && extension.bytes().all(|b| b.is_ascii_alphanumeric()) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitStatusFilter { + Modified, + Untracked, + Staged, + Unmodified, +} + +/// Buffer for text parts during query parsing. +pub(crate) type TextPartsBuffer<'a> = Vec<&'a str>; diff --git a/crates/fff-query-parser/src/glob_detect.rs b/crates/fff-query-parser/src/glob_detect.rs new file mode 100644 index 0000000..3a2fa04 --- /dev/null +++ b/crates/fff-query-parser/src/glob_detect.rs @@ -0,0 +1,18 @@ +//! Glob wildcard detection — delegates to zlob when available, pure-Rust fallback otherwise. +//! +//! All call sites use a single function: `has_wildcards(text) -> bool`. +//! When the `zlob` feature is enabled this calls `zlob::has_wildcards` with +//! `ZlobFlags::RECOMMENDED`; without it we check for the same set of wildcard +//! characters (`*`, `?`, `[`, `{`) in pure Rust. + +#[cfg(feature = "zlob")] +#[inline] +pub fn has_wildcards(s: &str) -> bool { + zlob::has_wildcards(s, zlob::ZlobFlags::RECOMMENDED) +} + +#[cfg(not(feature = "zlob"))] +#[inline] +pub fn has_wildcards(s: &str) -> bool { + s.bytes().any(|b| matches!(b, b'*' | b'?' | b'[' | b'{')) +} diff --git a/crates/fff-query-parser/src/lib.rs b/crates/fff-query-parser/src/lib.rs new file mode 100644 index 0000000..8dfcd4d --- /dev/null +++ b/crates/fff-query-parser/src/lib.rs @@ -0,0 +1,235 @@ +//! Fast query parser for file search +//! +//! This parser takes a search query and extracts structured constraints +//! while preserving text for fuzzy matching. Designed for maximum performance: +//! - Single-pass parsing with minimal branching +//! - Stack-allocated string buffers +//! +//! # Examples +//! +//! ``` +//! use fff_query_parser::{QueryParser, Constraint, FuzzyQuery}; +//! +//! let parser = QueryParser::default(); +//! +//! // Single-token queries return FFFQuery with Text fuzzy query and no constraints +//! let result = parser.parse("hello"); +//! assert!(result.constraints.is_empty()); +//! assert_eq!(result.fuzzy_query, FuzzyQuery::Text("hello")); +//! +//! // Multi-token queries are parsed +//! let result = parser.parse("name *.rs"); +//! match &result.fuzzy_query { +//! FuzzyQuery::Text(text) => assert_eq!(*text, "name"), +//! _ => panic!("Expected text"), +//! } +//! assert!(matches!(result.constraints[0], Constraint::Extension("rs"))); +//! +//! // Parse glob pattern with text +//! let result = parser.parse("**/*.rs foo"); +//! assert!(matches!(result.constraints[0], Constraint::Glob("**/*.rs"))); +//! +//! // Parse negation +//! let result = parser.parse("!*.rs foo"); +//! match &result.constraints[0] { +//! Constraint::Not(inner) => { +//! assert!(matches!(inner.as_ref(), Constraint::Extension("rs"))); +//! } +//! _ => panic!("Expected Not constraint"), +//! } +//! ``` + +mod config; +mod constraints; +pub mod glob_detect; +pub mod location; +mod parser; + +#[allow(deprecated)] +pub use config::is_filename_constraint_token; +pub use config::{ + AiGrepConfig, DirSearchConfig, FileSearchConfig, GrepConfig, MixedSearchConfig, ParserConfig, +}; +pub use constraints::{Constraint, GitStatusFilter}; +pub use location::Location; +pub use parser::{FFFQuery, FuzzyQuery, QueryParser}; + +pub type ConstraintVec<'a> = Vec>; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty_query() { + let parser = QueryParser::default(); + let result = parser.parse(""); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Empty); + } + + #[test] + fn test_whitespace_only() { + let parser = QueryParser::default(); + let result = parser.parse(" "); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Empty); + } + + #[test] + fn test_single_token() { + let parser = QueryParser::default(); + let result = parser.parse("hello"); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("hello")); + } + + #[test] + fn test_simple_text() { + let parser = QueryParser::default(); + let result = parser.parse("hello world"); + + match &result.fuzzy_query { + FuzzyQuery::Parts(parts) => { + assert_eq!(parts.len(), 2); + assert_eq!(parts[0], "hello"); + assert_eq!(parts[1], "world"); + } + _ => panic!("Expected Parts fuzzy query"), + } + + assert_eq!(result.constraints.len(), 0); + } + + #[test] + fn test_extension_only() { + let parser = QueryParser::default(); + // Single constraint token - returns Some so constraint can be applied + let result = parser.parse("*.rs"); + assert!(matches!(result.fuzzy_query, FuzzyQuery::Empty)); + assert_eq!(result.constraints.len(), 1); + assert!(matches!(result.constraints[0], Constraint::Extension("rs"))); + } + + #[test] + fn test_glob_pattern() { + let parser = QueryParser::default(); + let result = parser.parse("**/*.rs foo"); + assert_eq!(result.constraints.len(), 1); + // Glob patterns with ** are treated as globs, not extensions + match &result.constraints[0] { + Constraint::Glob(pattern) => assert_eq!(*pattern, "**/*.rs"), + other => panic!("Expected Glob constraint, got {:?}", other), + } + } + + #[test] + fn test_negation_pattern() { + let parser = QueryParser::default(); + let result = parser.parse("!test foo"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!(matches!(**inner, Constraint::Text("test"))); + } + _ => panic!("Expected Not constraint"), + } + } + + #[test] + fn test_path_segment() { + let parser = QueryParser::default(); + let result = parser.parse("/src/ foo"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::PathSegment("src") + )); + } + + #[test] + fn test_git_status() { + let parser = QueryParser::default(); + let result = parser.parse("status:modified foo"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::GitStatus(GitStatusFilter::Modified) + )); + } + + #[test] + fn test_file_type() { + let parser = QueryParser::default(); + let result = parser.parse("type:rust foo"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::FileType("rust") + )); + } + + #[test] + fn test_complex_query() { + let parser = QueryParser::default(); + let result = parser.parse("src name *.rs !test /lib/ status:modified"); + + // Verify we have fuzzy text + match &result.fuzzy_query { + FuzzyQuery::Parts(parts) => { + assert_eq!(parts.len(), 2); + assert_eq!(parts[0], "src"); + assert_eq!(parts[1], "name"); + } + _ => panic!("Expected Parts fuzzy query"), + } + + // Should have multiple constraints + assert!(result.constraints.len() >= 4); + + // Verify specific constraints exist + let has_extension = result + .constraints + .iter() + .any(|c| matches!(c, Constraint::Extension("rs"))); + let has_not = result + .constraints + .iter() + .any(|c| matches!(c, Constraint::Not(_))); + let has_path = result + .constraints + .iter() + .any(|c| matches!(c, Constraint::PathSegment("lib"))); + let has_git_status = result + .constraints + .iter() + .any(|c| matches!(c, Constraint::GitStatus(_))); + + assert!(has_extension, "Should have Extension constraint"); + assert!(has_not, "Should have Not constraint"); + assert!(has_path, "Should have PathSegment constraint"); + assert!(has_git_status, "Should have GitStatus constraint"); + } + + #[test] + fn test_small_constraint_count() { + let parser = QueryParser::default(); + let result = parser.parse("*.rs *.toml !test"); + assert_eq!(result.constraints.len(), 3); + } + + #[test] + fn test_many_fuzzy_parts() { + let parser = QueryParser::default(); + let result = parser.parse("one two three four five six"); + + match &result.fuzzy_query { + FuzzyQuery::Parts(parts) => { + assert_eq!(parts.len(), 6); + assert_eq!(parts[0], "one"); + assert_eq!(parts[5], "six"); + } + _ => panic!("Expected Parts fuzzy query"), + } + } +} diff --git a/crates/fff-query-parser/src/location.rs b/crates/fff-query-parser/src/location.rs new file mode 100644 index 0000000..9f951bb --- /dev/null +++ b/crates/fff-query-parser/src/location.rs @@ -0,0 +1,265 @@ +//! Location parsing for file:line:col patterns +//! +//! Parses various location formats like: +//! - `file:12` - Line number +//! - `file:12:4` - Line and column +//! - `file:12-114` - Line range +//! - `file:12:4-20` - Column range on same line +//! - `file:12:4-14:20` - Position range +//! - `file(12)` - Visual Studio style line +//! - `file(12,4)` - Visual Studio style line and column + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub enum Location { + Line(i32), + Range { start: (i32, i32), end: (i32, i32) }, + Position { line: i32, col: i32 }, +} + +fn parse_number_pair(location: &str, split_char: char) -> Option<(i32, i32)> { + let mut iter = location.split(split_char); + + let start_str = iter.next()?; + let end_str = iter.next()?; + + // if there are more than 2 parts it's not the range treat as normal query + if iter.next().is_some() { + return None; + } + + let start = start_str.parse::().ok()?; + let end = end_str.parse::().ok()?; + + Some((start, end)) +} + +/// Parse "line-line" format +fn parse_simple_range(location: &str) -> Option { + let (start, end) = parse_number_pair(location, '-')?; + if end < start { + return Some(Location::Line(start)); + } + + Some(Location::Range { + start: (start, 0), + end: (end, 0), + }) +} + +/// Parse "line:col-col" format (column range on same line) +fn parse_column_range(start_part: &str, end_part: &str) -> Option { + let (line_str, start_col_str) = start_part.split_once(':')?; + let line = line_str.parse::().ok()?; + let start_col = start_col_str.parse::().ok()?; + let end_col = end_part.parse::().ok()?; + + if end_col < start_col { + return Some(Location::Line(line)); + } + + Some(Location::Range { + start: (line, start_col), + end: (line, end_col), + }) +} + +/// Parse "line:col-line:col" format (position range) +fn parse_position_range(start_part: &str, end_part: &str) -> Option { + let (start_line, start_col) = parse_number_pair(start_part, ':')?; + let (end_line, end_col) = parse_number_pair(end_part, ':')?; + + if end_line < start_line || (end_line == start_line && end_col < start_col) { + return Some(Location::Position { + line: start_line, + col: start_col, + }); + } + + Some(Location::Range { + start: (start_line, start_col), + end: (end_line, end_col), + }) +} + +/// Try to parse range patterns (contains '-') +fn try_parse_column_range(location: &str) -> Option { + if !location.contains('-') { + return None; + } + + let (start_part, end_part) = location.split_once('-')?; + + // Try position range (line:col-line:col) + if start_part.contains(':') && end_part.contains(':') { + return parse_position_range(start_part, end_part); + } + + // Try column range (line:col-col) + if start_part.contains(':') { + return parse_column_range(start_part, end_part); + } + + // Try simple line range (line-line) + parse_simple_range(location) +} + +/// Try to parse position patterns (contains ':' but not '-') +fn try_parse_column_position(location: &str) -> Option { + if !location.contains(':') { + return None; + } + + let (line_str, col_str) = location.split_once(':')?; + let line = line_str.parse::().ok()?; + let col = col_str.parse::().ok()?; + + Some(Location::Position { line, col }) +} + +/// Parses various location formats like file:12, file:12:4, file:12-114 +fn parse_column_location(query: &str) -> Option<(&str, Location)> { + let (file_path, location_part) = query.split_once(':')?; + + if let Some(range_location) = try_parse_column_range(location_part) { + return Some((file_path, range_location)); + } + + if let Some(position_location) = try_parse_column_position(location_part) { + return Some((file_path, position_location)); + } + + if let Ok(line_location) = location_part.parse::() { + return Some((file_path, Location::Line(line_location))); + } + + None +} + +fn parse_vstudio_location(query: &str) -> Option<(&str, Location)> { + if !query.ends_with(')') { + return None; + } + + let (file_path, location_with_paren) = query.rsplit_once('(')?; + let location = location_with_paren.trim_end_matches(')'); + + if let Ok(line) = location.parse::() { + return Some((file_path, Location::Line(line))); + } + + if let Some((line, col)) = parse_number_pair(location, ',') { + return Some((file_path, Location::Position { line, col })); + } + + None +} + +/// Parse location from the end of a query string. +/// +/// Returns the query without the location suffix, and the parsed location if found. +/// +/// # Examples +/// ``` +/// use fff_query_parser::location::{parse_location, Location}; +/// +/// let (query, loc) = parse_location("file:12"); +/// assert_eq!(query, "file"); +/// assert_eq!(loc, Some(Location::Line(12))); +/// +/// let (query, loc) = parse_location("search term"); +/// assert_eq!(query, "search term"); +/// assert_eq!(loc, None); +/// ``` +pub fn parse_location(query: &str) -> (&str, Option) { + // simply ignore the last semicolon even if there are no additional location info + let query = query.trim_end_matches([':', '-', '(']); + if let Some((path, location)) = parse_column_location(query) { + return (path, Some(location)); + } + + if let Some((path, location)) = parse_vstudio_location(query) { + return (path, Some(location)); + } + + (query, None) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_location_parsing() { + assert_eq!( + parse_location("new_file:12"), + ("new_file", Some(Location::Line(12))) + ); + assert_eq!(parse_location("new_file:12ab"), ("new_file:12ab", None)); + + assert_eq!(parse_location("something"), ("something", None)); + assert_eq!( + parse_location("file:12:4"), + ("file", Some(Location::Position { line: 12, col: 4 })) + ); + + assert_eq!( + parse_location("file:12-114"), + ( + "file", + Some(Location::Range { + start: (12, 0), + end: (114, 0) + }) + ) + ); + + assert_eq!( + parse_location("file:12:4-20"), + ( + "file", + Some(Location::Range { + start: (12, 4), + end: (12, 20) + }) + ) + ); + + assert_eq!( + parse_location("file:100:4-14:20"), + ("file", Some(Location::Position { line: 100, col: 4 })) + ); + + assert_eq!( + parse_location("file:12:4-14:20"), + ( + "file", + Some(Location::Range { + start: (12, 4), + end: (14, 20) + }) + ) + ); + } + + #[test] + fn test_vstudio_parsing() { + assert_eq!( + parse_location("file(12)"), + ("file", Some(Location::Line(12))) + ); + assert_eq!( + parse_location("file(12,4)"), + ("file", Some(Location::Position { line: 12, col: 4 })) + ); + } + + #[test] + fn trimes_end_character() { + assert_eq!( + parse_location("file:12-"), + ("file", Some(Location::Line(12))) + ); + assert_eq!(parse_location("file:-"), ("file", None)); + assert_eq!(parse_location("file("), ("file", None)); + } +} diff --git a/crates/fff-query-parser/src/parser.rs b/crates/fff-query-parser/src/parser.rs new file mode 100644 index 0000000..88a32e9 --- /dev/null +++ b/crates/fff-query-parser/src/parser.rs @@ -0,0 +1,1467 @@ +use crate::ConstraintVec; +use crate::config::ParserConfig; +use crate::constraints::{Constraint, GitStatusFilter, TextPartsBuffer}; +use crate::glob_detect::has_wildcards; +use crate::location::{Location, parse_location}; + +#[derive(Debug, Clone, PartialEq)] +#[allow(clippy::large_enum_variant)] +pub enum FuzzyQuery<'a> { + Parts(TextPartsBuffer<'a>), + Text(&'a str), + Empty, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FFFQuery<'a> { + /// The original raw query string before parsing + pub raw_query: &'a str, + /// Parsed constraints (stack-allocated for ≤8 constraints) + pub constraints: ConstraintVec<'a>, + pub fuzzy_query: FuzzyQuery<'a>, + /// Parsed location (e.g., file:12:4 -> line 12, col 4) + pub location: Option, +} + +impl<'a> FFFQuery<'a> { + /// Parse query and execute, perfectly paired with configuration presets + /// + /// ``` + /// use fff_query_parser::{FFFQuery, FileSearchConfig}; + /// + /// let query = FFFQuery::parse("file *.rs", FileSearchConfig); + /// ``` + pub fn parse(query: &'a str, config: impl ParserConfig) -> Self { + let query_parser = QueryParser::new(config); + + query_parser.parse(query.as_ref()) + } +} + +/// Main query parser - zero-cost wrapper around configuration +#[derive(Debug)] +pub struct QueryParser { + config: C, +} + +impl QueryParser { + pub fn new(config: C) -> Self { + Self { config } + } + + pub fn parse<'a>(&self, query: &'a str) -> FFFQuery<'a> { + let raw_query = query; + let config: &C = &self.config; + let mut constraints = ConstraintVec::new(); + let query = query.trim(); + + let whitespace_count = query.chars().filter(|c| c.is_whitespace()).count(); + + // Single token - check if it's a constraint or plain text + if whitespace_count == 0 { + // Try to parse as constraint first + if let Some(constraint) = parse_token(query, config) { + // Don't treat filename tokens (FilePath) as constraints in single-token + // queries — the user is fuzzy-searching, not filtering. FilePath constraints + // are only useful as filters in multi-token queries like "score.rs search". + // + // Also skip PathSegment constraints when the token looks like an absolute + // file path with a location suffix (e.g. /Users/.../file.rs:12). Without + // this, the leading `/` causes the entire path to be consumed as a + // PathSegment, preventing location parsing from running. + let has_location_suffix = matches!(constraint, Constraint::PathSegment(_)) + && query.bytes().any(|b| b == b':') + && query + .bytes() + .rev() + .take_while(|&b| b != b':') + .all(|b| b.is_ascii_digit()); + + // for grep we don't want to treat a part of path like pathname + let treat_as_text = matches!(constraint, Constraint::PathSegment(_)) + && config.treat_lone_path_as_text(); + + if !matches!(constraint, Constraint::FilePath(_)) + && !has_location_suffix + && !treat_as_text + { + constraints.push(constraint); + return FFFQuery { + raw_query, + constraints, + fuzzy_query: FuzzyQuery::Empty, + location: None, + }; + } + } + + // Try to extract location from single token (e.g., "file:12") + if config.enable_location() { + let (query_without_loc, location) = parse_location(query); + if location.is_some() { + return FFFQuery { + raw_query, + constraints, + fuzzy_query: FuzzyQuery::Text(query_without_loc), + location, + }; + } + } + + // Plain text single token + return FFFQuery { + raw_query, + constraints, + fuzzy_query: if query.is_empty() { + FuzzyQuery::Empty + } else { + FuzzyQuery::Text(query) + }, + location: None, + }; + } + + let mut text_parts = TextPartsBuffer::new(); + let tokens = query.split_whitespace(); + + let mut has_file_path = false; + // Track the FilePath token position in constraints so we can promote + // it back to text if the final query ends up with no fuzzy text. + let mut file_path_constraint_idx: Option = None; + let mut file_path_token: Option<&str> = None; + for token in tokens { + match parse_token(token, config) { + Some(Constraint::FilePath(_)) => { + if has_file_path { + // Only one FilePath constraint allowed; treat extra path + // tokens as literal text (e.g. an import path the user is + // searching for). + text_parts.push(token); + } else { + file_path_constraint_idx = Some(constraints.len()); + file_path_token = Some(token); + constraints.push(Constraint::FilePath(token)); + has_file_path = true; + } + } + Some(constraint) => { + constraints.push(constraint); + } + None => { + text_parts.push(token); + } + } + } + + // If the query produced a single FilePath and no fuzzy text parts, the + // user isn't filtering by filename suffix — they're fuzzy-searching + // for that name (the only other constraints are path-scoping like + // PathSegment/Extension/Glob). Mirror the single-token rule at + // parser.rs:48-64: promote FilePath → fuzzy text so e.g. `profile.h` + // alongside `chrome/browser/profiles/` fuzzy-matches all `profile*.h` + // files instead of only one file literally ending in `/profile.h`. + if text_parts.is_empty() + && let Some(idx) = file_path_constraint_idx + && let Some(tok) = file_path_token + { + constraints.remove(idx); + text_parts.push(tok); + } + + // Try to extract location from the last fuzzy token + // e.g., "search file:12" -> fuzzy="search file", location=Line(12) + let location = if config.enable_location() && !text_parts.is_empty() { + let last_idx = text_parts.len() - 1; + let (without_loc, loc) = parse_location(text_parts[last_idx]); + if loc.is_some() { + // Update the last part to be without the location suffix + text_parts[last_idx] = without_loc; + loc + } else { + None + } + } else { + None + }; + + let fuzzy_query = if text_parts.is_empty() { + FuzzyQuery::Empty + } else if text_parts.len() == 1 { + // If the only remaining text is empty after location extraction, treat as Empty + if text_parts[0].is_empty() { + FuzzyQuery::Empty + } else { + FuzzyQuery::Text(text_parts[0]) + } + } else { + // Filter out empty parts that might result from location extraction + if text_parts.iter().all(|p| p.is_empty()) { + FuzzyQuery::Empty + } else { + FuzzyQuery::Parts(text_parts) + } + }; + + FFFQuery { + raw_query, + constraints, + fuzzy_query, + location, + } + } +} + +impl<'a> FFFQuery<'a> { + /// Returns the grep search text by joining all non-constraint text tokens. + /// + /// Backslash-escaped tokens (e.g. `\*.rs`) are included as literal text + /// with the leading `\` stripped, since the backslash is only an escape + /// signal to the parser and should not appear in the final pattern. + /// + /// `FuzzyQuery::Empty` → empty string + /// `FuzzyQuery::Text("foo")` → `"foo"` + /// `FuzzyQuery::Parts(["a", "\\*.rs", "b"])` → `"a *.rs b"` + pub fn grep_text(&self) -> String { + match &self.fuzzy_query { + FuzzyQuery::Empty => String::new(), + FuzzyQuery::Text(t) => strip_leading_backslash(t).to_string(), + FuzzyQuery::Parts(parts) => parts + .iter() + .map(|t| strip_leading_backslash(t)) + .collect::>() + .join(" "), + } + } +} + +/// Strip the leading `\` from a backslash-escaped constraint token only. +/// +/// We strip the backslash when the next character is a constraint trigger +/// (`*`, `/`, `!`) — the user typed `\*.rs` to mean literal `*.rs`, not an +/// extension constraint. For regex escape sequences like `\w`, `\b`, `\d`, +/// `\s`, `\n` etc., the backslash is preserved so regex mode works correctly. +#[inline] +fn strip_leading_backslash(token: &str) -> &str { + if token.len() > 1 && token.starts_with('\\') { + let next = token.as_bytes()[1]; + // Only strip if the backslash is escaping a constraint trigger character + if next == b'*' || next == b'/' || next == b'!' { + return &token[1..]; + } + } + token +} + +impl Default for QueryParser { + fn default() -> Self { + Self::new(crate::FileSearchConfig) + } +} + +#[inline] +fn parse_token<'a, C: ParserConfig>(token: &'a str, config: &C) -> Option> { + // Backslash escape: \token → treat as literal text, skip all constraint parsing. + // The leading \ is stripped by the caller when building the search text. + if token.starts_with('\\') && token.len() > 1 { + return None; + } + + let first_byte = token.as_bytes().first()?; + + match first_byte { + b'*' if config.enable_extension() => { + // Ignore incomplete patterns like "*" or "*." + if token == "*" || token == "*." { + return None; + } + + // Try extension first (*.rs) - simple patterns without additional wildcards + if let Some(constraint) = parse_extension(token) { + // Only return Extension if the rest doesn't have wildcards + // e.g., *.rs is Extension, but *.test.* should be Glob + let ext_part = &token[2..]; + if !has_wildcards(ext_part) { + return Some(constraint); + } + } + // Has wildcards -> use config-specific glob detection + if config.enable_glob() && config.is_glob_pattern(token) { + return Some(Constraint::Glob(token)); + } + None + } + b'!' if config.enable_exclude() => parse_negation(token, config), + b'/' if config.enable_path_segments() => parse_path_segment(token), + // Handle trailing slash syntax: www/ -> PathSegment("www") + _ if config.enable_path_segments() && token.ends_with('/') => { + parse_path_segment_trailing(token) + } + // tokens like `file.rs` *if enabled* + _ if config.enable_filename_constraint() + && !token.ends_with('/') + && Constraint::is_filename_constraint_token(token) => + { + Some(Constraint::FilePath(token)) + } + _ => { + // Check for glob patterns using config-specific detection + if config.enable_glob() && config.is_glob_pattern(token) { + return Some(Constraint::Glob(token)); + } + + // Check for key:value patterns + if let Some(colon_idx) = memchr(b':', token.as_bytes()) { + let (key, value_with_colon) = token.split_at(colon_idx); + let value = &value_with_colon[1..]; // Skip the colon + + match key { + "type" if config.enable_type_filter() => { + return Some(Constraint::FileType(value)); + } + "status" | "st" | "g" | "git" if config.enable_git_status() => { + return parse_git_status(value); + } + _ => {} + } + } + + // Try custom parsers + config.parse_custom(token) + } + } +} + +/// Scalar byte find. Queries are tiny (<100 bytes), so this stays dependency-free +/// instead of pulling in the `memchr` crate; fff-core uses `memchr` for real haystacks. +#[inline] +fn memchr(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == needle) +} + +/// Parse extension pattern: *.rs -> Extension("rs") +#[inline] +fn parse_extension(token: &str) -> Option> { + if token.len() > 2 && token.starts_with("*.") { + Some(Constraint::Extension(&token[2..])) + } else { + None + } +} + +/// Parse negation pattern: !*.rs -> Not(Extension("rs")), !test -> Not(Text("test")) +/// This allows negating any constraint type +#[inline] +fn parse_negation<'a, C: ParserConfig>(token: &'a str, config: &C) -> Option> { + if token.len() <= 1 { + return None; + } + + let inner_token = &token[1..]; + + // Try to parse the inner token as any constraint + if let Some(inner_constraint) = parse_token_without_negation(inner_token, config) { + // Wrap it in a Not constraint + return Some(Constraint::Not(Box::new(inner_constraint))); + } + + // If it's not a special constraint, treat it as negated text + // For backward compatibility with !test syntax + Some(Constraint::Not(Box::new(Constraint::Text(inner_token)))) +} + +/// Parse a token without checking for negation (to avoid infinite recursion) +#[inline] +fn parse_token_without_negation<'a, C: ParserConfig>( + token: &'a str, + config: &C, +) -> Option> { + // Backslash escape applies here too + if token.starts_with('\\') && token.len() > 1 { + return None; + } + + let first_byte = token.as_bytes().first()?; + + match first_byte { + b'*' if config.enable_extension() => { + // Try extension first (*.rs) - simple patterns without additional wildcards + if let Some(constraint) = parse_extension(token) { + let ext_part = &token[2..]; + if !has_wildcards(ext_part) { + return Some(constraint); + } + } + // Has wildcards -> use config-specific glob detection + if config.enable_glob() && config.is_glob_pattern(token) { + return Some(Constraint::Glob(token)); + } + None + } + b'/' if config.enable_path_segments() => parse_path_segment(token), + _ if config.enable_path_segments() && token.ends_with('/') => { + // Handle trailing slash syntax: www/ -> PathSegment("www") + parse_path_segment_trailing(token) + } + _ => { + // Check for glob patterns using config-specific detection + if config.enable_glob() && config.is_glob_pattern(token) { + return Some(Constraint::Glob(token)); + } + + // Check for key:value patterns + if let Some(colon_idx) = memchr(b':', token.as_bytes()) { + let (key, value_with_colon) = token.split_at(colon_idx); + let value = &value_with_colon[1..]; // Skip the colon + + match key { + "type" if config.enable_type_filter() => { + return Some(Constraint::FileType(value)); + } + "status" | "st" | "g" | "git" if config.enable_git_status() => { + return parse_git_status(value); + } + _ => {} + } + } + + if config.enable_filename_constraint() + && Constraint::is_filename_constraint_token(token) + { + return Some(Constraint::FilePath(token)); + } + + config.parse_custom(token) + } + } +} + +/// Parse path segment: /src/ -> PathSegment("src") +#[inline] +fn parse_path_segment(token: &str) -> Option> { + if token.len() > 1 && token.starts_with('/') { + let segment = token.trim_start_matches('/').trim_end_matches('/'); + if !segment.is_empty() { + Some(Constraint::PathSegment(segment)) + } else { + None + } + } else { + None + } +} + +/// Parse path segment with trailing slash: www/ -> PathSegment("www") +/// Also supports multi-segment paths: libswscale/aarch64/ -> PathSegment("libswscale/aarch64") +#[inline] +fn parse_path_segment_trailing(token: &str) -> Option> { + if token.len() > 1 && token.ends_with('/') { + let segment = token.trim_end_matches('/'); + if !segment.is_empty() { + Some(Constraint::PathSegment(segment)) + } else { + None + } + } else { + None + } +} + +/// Parse git status filter: modified|m|untracked|u|staged|s +#[inline] +fn parse_git_status(value: &str) -> Option> { + if value == "*" { + return None; + } + + if "modified".starts_with(value) { + return Some(Constraint::GitStatus(GitStatusFilter::Modified)); + } + + if "untracked".starts_with(value) { + return Some(Constraint::GitStatus(GitStatusFilter::Untracked)); + } + + if "staged".starts_with(value) { + return Some(Constraint::GitStatus(GitStatusFilter::Staged)); + } + + if "clean".starts_with(value) { + return Some(Constraint::GitStatus(GitStatusFilter::Unmodified)); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{FileSearchConfig, GrepConfig}; + + /// File-picker-like config with filename-constraint detection enabled, + /// mirroring the Neovim layer's opt-in behavior. + struct FilenameConstraintConfig; + + impl ParserConfig for FilenameConstraintConfig { + fn enable_filename_constraint(&self) -> bool { + true + } + } + + #[test] + fn test_parse_extension() { + assert_eq!(parse_extension("*.rs"), Some(Constraint::Extension("rs"))); + assert_eq!( + parse_extension("*.toml"), + Some(Constraint::Extension("toml")) + ); + assert_eq!(parse_extension("*"), None); + assert_eq!(parse_extension("*."), None); + } + + #[test] + fn test_incomplete_patterns_ignored() { + let config = FileSearchConfig; + // Incomplete patterns should return None and be treated as noise + assert_eq!(parse_token("*", &config), None); + assert_eq!(parse_token("*.", &config), None); + } + + #[test] + fn test_parse_path_segment() { + assert_eq!( + parse_path_segment("/src/"), + Some(Constraint::PathSegment("src")) + ); + assert_eq!( + parse_path_segment("/lib"), + Some(Constraint::PathSegment("lib")) + ); + assert_eq!(parse_path_segment("/"), None); + } + + #[test] + fn test_parse_path_segment_trailing() { + assert_eq!( + parse_path_segment_trailing("www/"), + Some(Constraint::PathSegment("www")) + ); + assert_eq!( + parse_path_segment_trailing("src/"), + Some(Constraint::PathSegment("src")) + ); + // Multi-segment paths should work + assert_eq!( + parse_path_segment_trailing("src/lib/"), + Some(Constraint::PathSegment("src/lib")) + ); + assert_eq!( + parse_path_segment_trailing("libswscale/aarch64/"), + Some(Constraint::PathSegment("libswscale/aarch64")) + ); + // Should not match without trailing slash + assert_eq!(parse_path_segment_trailing("www"), None); + } + + #[test] + fn test_trailing_slash_in_query() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("www/ test"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::PathSegment("www") + )); + assert!(matches!(result.fuzzy_query, FuzzyQuery::Text("test"))); + } + + #[test] + fn test_parse_git_status() { + assert_eq!( + parse_git_status("modified"), + Some(Constraint::GitStatus(GitStatusFilter::Modified)) + ); + assert_eq!( + parse_git_status("m"), + Some(Constraint::GitStatus(GitStatusFilter::Modified)) + ); + assert_eq!( + parse_git_status("untracked"), + Some(Constraint::GitStatus(GitStatusFilter::Untracked)) + ); + assert_eq!(parse_git_status("invalid"), None); + } + + #[test] + fn test_memchr() { + assert_eq!(memchr(b':', b"type:rust"), Some(4)); + assert_eq!(memchr(b':', b"nocolon"), None); + assert_eq!(memchr(b':', b":start"), Some(0)); + } + + #[test] + fn test_negation_text() { + let parser = QueryParser::new(FileSearchConfig); + // Need two tokens for parsing to return Some + let result = parser.parse("!test foo"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!(matches!(**inner, Constraint::Text("test"))); + } + _ => panic!("Expected Not constraint"), + } + } + + #[test] + fn test_negation_extension() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("!*.rs foo"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!(matches!(**inner, Constraint::Extension("rs"))); + } + _ => panic!("Expected Not(Extension) constraint"), + } + } + + #[test] + fn test_negation_path_segment() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("!/src/ foo"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!(matches!(**inner, Constraint::PathSegment("src"))); + } + _ => panic!("Expected Not(PathSegment) constraint"), + } + } + + #[test] + fn test_negation_git_status() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("!status:modified foo"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!(matches!( + **inner, + Constraint::GitStatus(GitStatusFilter::Modified) + )); + } + _ => panic!("Expected Not(GitStatus) constraint"), + } + } + + #[test] + fn test_negation_git_status_all_key_aliases() { + let parser = QueryParser::new(FileSearchConfig); + for key in ["status", "st", "g", "git"] { + let query = format!("!{key}:modified foo"); + let result = parser.parse(&query); + assert_eq!( + result.constraints.len(), + 1, + "!{key}:modified should produce exactly one constraint" + ); + match &result.constraints[0] { + Constraint::Not(inner) => assert!( + matches!(**inner, Constraint::GitStatus(GitStatusFilter::Modified)), + "!{key}:modified expected Not(GitStatus(Modified)), got Not({inner:?})" + ), + other => { + panic!("!{key}:modified expected Not(GitStatus), got {other:?}") + } + } + } + } + + #[test] + fn test_backslash_escape_extension() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("\\*.rs foo"); + // \*.rs should NOT be parsed as an Extension constraint + assert_eq!(result.constraints.len(), 0); + // Both tokens should be text + match result.fuzzy_query { + FuzzyQuery::Parts(parts) => { + assert_eq!(parts.len(), 2); + assert_eq!(parts[0], "\\*.rs"); + assert_eq!(parts[1], "foo"); + } + _ => panic!("Expected Parts, got {:?}", result.fuzzy_query), + } + } + + #[test] + fn test_backslash_escape_path_segment() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("\\/src/ foo"); + assert_eq!(result.constraints.len(), 0); + match result.fuzzy_query { + FuzzyQuery::Parts(parts) => { + assert_eq!(parts[0], "\\/src/"); + assert_eq!(parts[1], "foo"); + } + _ => panic!("Expected Parts, got {:?}", result.fuzzy_query), + } + } + + #[test] + fn test_backslash_escape_negation() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("\\!test foo"); + assert_eq!(result.constraints.len(), 0); + } + + #[test] + fn test_grep_text_plain_text() { + // Multi-token plain text — no constraints + let q = QueryParser::new(GrepConfig).parse("name ="); + assert_eq!(q.grep_text(), "name ="); + } + + #[test] + fn test_grep_text_strips_constraint() { + let q = QueryParser::new(GrepConfig).parse("name = *.rs someth"); + assert_eq!(q.grep_text(), "name = someth"); + } + + #[test] + fn test_grep_text_leading_constraint() { + let q = QueryParser::new(GrepConfig).parse("*.rs name ="); + assert_eq!(q.grep_text(), "name ="); + } + + #[test] + fn test_grep_text_only_constraints() { + let q = QueryParser::new(GrepConfig).parse("*.rs /src/"); + assert_eq!(q.grep_text(), ""); + } + + #[test] + fn test_grep_text_path_constraint() { + let q = QueryParser::new(GrepConfig).parse("name /src/ value"); + assert_eq!(q.grep_text(), "name value"); + } + + #[test] + fn test_grep_text_negation_constraint() { + let q = QueryParser::new(GrepConfig).parse("name !*.rs value"); + assert_eq!(q.grep_text(), "name value"); + } + + #[test] + fn test_grep_text_backslash_escape_stripped() { + // \*.rs should be text with the leading \ removed + let q = QueryParser::new(GrepConfig).parse("\\*.rs foo"); + assert_eq!(q.grep_text(), "*.rs foo"); + + let q = QueryParser::new(GrepConfig).parse("\\/src/ foo"); + assert_eq!(q.grep_text(), "/src/ foo"); + + let q = QueryParser::new(GrepConfig).parse("\\!test foo"); + assert_eq!(q.grep_text(), "!test foo"); + } + + #[test] + fn test_grep_text_question_mark_is_text() { + let q = QueryParser::new(GrepConfig).parse("foo? bar"); + assert_eq!(q.grep_text(), "foo? bar"); + } + + #[test] + fn test_grep_text_bracket_is_text() { + let q = QueryParser::new(GrepConfig).parse("arr[0] more"); + assert_eq!(q.grep_text(), "arr[0] more"); + } + + #[test] + fn test_grep_text_path_glob_is_constraint() { + let q = QueryParser::new(GrepConfig).parse("pattern src/**/*.rs"); + assert_eq!(q.grep_text(), "pattern"); + } + + #[test] + fn test_grep_question_mark_is_text() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("foo?"); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("foo?")); + } + + #[test] + fn test_grep_bracket_is_text() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("arr[0] something"); + // arr[0] should NOT be a glob in grep mode + assert_eq!(result.constraints.len(), 0); + } + + #[test] + fn test_grep_path_glob_is_constraint() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("pattern src/**/*.rs"); + // src/**/*.rs contains / so it should be treated as a glob + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::Glob("src/**/*.rs") + )); + } + + #[test] + fn test_grep_brace_is_constraint() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("pattern {src,lib}"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::Glob("{src,lib}") + )); + } + + #[test] + fn test_grep_text_preserves_backslash_escapes() { + // Regex patterns like \w+ and \bfoo\b must survive grep_text() + // The parser sees \w+ as a text token (not a constraint escape), + // but strip_leading_backslash was stripping the \ anyway. + let q = QueryParser::new(GrepConfig).parse("pub struct \\w+"); + assert_eq!( + q.grep_text(), + "pub struct \\w+", + "Backslash-w in regex must be preserved" + ); + + let q = QueryParser::new(GrepConfig).parse("\\bword\\b more"); + assert_eq!( + q.grep_text(), + "\\bword\\b more", + "Backslash-b word boundaries must be preserved" + ); + + // Single-token regex like "fn\\s+\\w+" returns FFFQuery with Text fuzzy query + let result = QueryParser::new(GrepConfig).parse("fn\\s+\\w+"); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("fn\\s+\\w+")); + + // But the escaped constraint forms SHOULD still be stripped: + let q = QueryParser::new(GrepConfig).parse("\\*.rs foo"); + assert_eq!( + q.grep_text(), + "*.rs foo", + "Escaped constraint \\*.rs should still have backslash stripped" + ); + + let q = QueryParser::new(GrepConfig).parse("\\/src/ foo"); + assert_eq!( + q.grep_text(), + "/src/ foo", + "Escaped constraint \\/src/ should still have backslash stripped" + ); + } + + #[test] + fn test_grep_bare_star_is_text() { + let parser = QueryParser::new(GrepConfig); + // "a*b" contains * but no / or {} — should be text in grep mode + let result = parser.parse("a*b something"); + assert_eq!( + result.constraints.len(), + 0, + "bare * without / should be text" + ); + } + + #[test] + fn test_grep_negated_text() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("pattern !test"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!( + matches!(**inner, Constraint::Text("test")), + "Expected Not(Text(\"test\")), got Not({:?})", + inner + ); + } + other => panic!("Expected Not constraint, got {:?}", other), + } + } + + #[test] + fn test_grep_negated_path_segment() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("pattern !/src/"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!( + matches!(**inner, Constraint::PathSegment("src")), + "Expected Not(PathSegment(\"src\")), got Not({:?})", + inner + ); + } + other => panic!("Expected Not constraint, got {:?}", other), + } + } + + #[test] + fn test_grep_negated_extension() { + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("pattern !*.rs"); + assert_eq!(result.constraints.len(), 1); + match &result.constraints[0] { + Constraint::Not(inner) => { + assert!( + matches!(**inner, Constraint::Extension("rs")), + "Expected Not(Extension(\"rs\")), got Not({:?})", + inner + ); + } + other => panic!("Expected Not constraint, got {:?}", other), + } + } + + #[test] + fn test_ai_grep_detects_file_path() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("libswscale/input.c rgba32ToY"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!( + result.constraints[0], + Constraint::FilePath("libswscale/input.c") + ), + "Expected FilePath, got {:?}", + result.constraints[0] + ); + assert_eq!(result.grep_text(), "rgba32ToY"); + } + + #[test] + fn test_ai_grep_detects_nested_file_path() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("src/main.rs fn main"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::FilePath("src/main.rs") + )); + assert_eq!(result.grep_text(), "fn main"); + } + + #[test] + fn test_ai_grep_no_false_positive_trailing_slash() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("src/ pattern"); + // Should be PathSegment, NOT FilePath + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::PathSegment("src")), + "Expected PathSegment, got {:?}", + result.constraints[0] + ); + } + + #[test] + fn test_ai_grep_bare_filename_is_file_path() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("main.rs pattern"); + // Bare filename with valid extension → FilePath constraint + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::FilePath("main.rs")), + "Expected FilePath, got {:?}", + result.constraints[0] + ); + assert_eq!(result.grep_text(), "pattern"); + } + + #[test] + fn test_ai_grep_filename_with_pathsegment_only_promotes_to_text() { + // When the ONLY non-text constraints are path-scoping (PathSegment, + // here), a bare filename token like `profile.h` should NOT be used as + // a FilePath filter — the user is fuzzy-searching within that dir, + // not asking for files named exactly `profile.h`. + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("chrome/browser/profiles/ profile.h"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!( + result.constraints[0], + Constraint::PathSegment("chrome/browser/profiles") + ), + "Expected single PathSegment, got {:?}", + result.constraints + ); + assert_eq!(result.grep_text(), "profile.h"); + } + + #[test] + fn test_ai_grep_leading_slash_path_alone_is_text_not_path_segment() { + // A leading-slash multi-segment path like `/api/tests/` or `/api/tests` + // used as the sole query token should be treated as fuzzy text, NOT as + // a PathSegment constraint. The user is searching for files matching + // that path string, not trying to scope results to a directory. + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + + // With trailing slash + let result = parser.parse("/api/tests/"); + assert_eq!( + result.constraints.len(), + 0, + "Expected no constraints for '/api/tests/', got {:?}", + result.constraints + ); + assert!( + matches!(result.fuzzy_query, FuzzyQuery::Text("/api/tests/")), + "Expected FuzzyQuery::Text, got {:?}", + result.fuzzy_query + ); + + // Without trailing slash + let result = parser.parse("/api/tests"); + assert_eq!( + result.constraints.len(), + 0, + "Expected no constraints for '/api/tests', got {:?}", + result.constraints + ); + assert!( + matches!(result.fuzzy_query, FuzzyQuery::Text("/api/tests")), + "Expected FuzzyQuery::Text, got {:?}", + result.fuzzy_query + ); + } + + #[test] + fn test_grep_leading_slash_path_alone_is_text_not_path_segment() { + // Same behavior for regular GrepConfig — single-token path-like + // queries are search terms, not directory filters. + let parser = QueryParser::new(GrepConfig); + + let result = parser.parse("/api/tests/"); + assert_eq!( + result.constraints.len(), + 0, + "GrepConfig: expected no constraints for '/api/tests/', got {:?}", + result.constraints + ); + assert!( + matches!(result.fuzzy_query, FuzzyQuery::Text("/api/tests/")), + "GrepConfig: expected FuzzyQuery::Text, got {:?}", + result.fuzzy_query + ); + + let result = parser.parse("/api/tests"); + assert_eq!( + result.constraints.len(), + 0, + "GrepConfig: expected no constraints for '/api/tests', got {:?}", + result.constraints + ); + assert!( + matches!(result.fuzzy_query, FuzzyQuery::Text("/api/tests")), + "GrepConfig: expected FuzzyQuery::Text, got {:?}", + result.fuzzy_query + ); + } + + #[test] + fn test_ai_grep_filename_with_extension_only_promotes_to_text() { + // Same case with an Extension constraint — no fuzzy text means the + // filename is what the user is searching for. + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("*.h profile.h"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::Extension("h")), + "Expected Extension, got {:?}", + result.constraints + ); + assert_eq!(result.grep_text(), "profile.h"); + } + + #[test] + fn test_ai_grep_filename_with_other_text_keeps_filepath() { + // Sanity: when there IS fuzzy text alongside the filename, the + // filename stays a FilePath filter (the documented multi-token case). + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("main.rs pattern"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::FilePath("main.rs")), + "Expected FilePath, got {:?}", + result.constraints + ); + assert_eq!(result.grep_text(), "pattern"); + } + + #[test] + fn test_ai_grep_bare_filename_schema_rs() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("schema.rs part_revisions"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::FilePath("schema.rs")), + "Expected FilePath(schema.rs), got {:?}", + result.constraints[0] + ); + assert_eq!(result.grep_text(), "part_revisions"); + } + + #[test] + fn test_ai_grep_bare_word_no_extension_not_constraint() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("schema pattern"); + // No extension → not a file path, just text + assert_eq!(result.constraints.len(), 0); + assert_eq!(result.grep_text(), "schema pattern"); + } + + #[test] + fn test_ai_grep_no_false_positive_no_extension() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("src/utils pattern"); + // No extension in last component → not a file path, just text + assert_eq!(result.constraints.len(), 0); + assert_eq!(result.grep_text(), "src/utils pattern"); + } + + #[test] + fn test_ai_grep_wildcard_not_filepath() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("src/**/*.rs pattern"); + // Contains wildcards → should be a Glob, not FilePath + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::Glob("src/**/*.rs")), + "Expected Glob, got {:?}", + result.constraints[0] + ); + } + + #[test] + fn test_ai_grep_star_text_star_is_glob() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("*quote* TODO"); + // `*quote*` should be recognised as a glob constraint in AI mode + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::Glob("*quote*")), + "Expected Glob(*quote*), got {:?}", + result.constraints[0] + ); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("TODO")); + } + + #[test] + fn test_ai_grep_bare_star_not_glob() { + use crate::AiGrepConfig; + let parser = QueryParser::new(AiGrepConfig); + let result = parser.parse("* pattern"); + // Bare `*` should NOT be treated as a glob (too broad) + assert!( + result.constraints.is_empty(), + "Expected no constraints, got {:?}", + result.constraints + ); + } + + #[test] + fn test_grep_no_location_parsing_single_token() { + let parser = QueryParser::new(GrepConfig); + // localhost:8080 should NOT be parsed as location -- it's a search pattern + let result = parser.parse("localhost:8080"); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("localhost:8080")); + } + + #[test] + fn test_grep_no_location_parsing_multi_token() { + let q = QueryParser::new(GrepConfig).parse("*.rs localhost:8080"); + assert_eq!( + q.grep_text(), + "localhost:8080", + "Colon-number suffix should be preserved in grep text" + ); + assert!( + q.location.is_none(), + "Grep should not parse location from colon-number" + ); + } + + #[test] + fn test_grep_reversed_braces_does_not_panic() { + // BUG PINING https://github.com/dmtrKovalenko/fff/issues/479 + // we should support any combination of different brackets without crashes + for query in [ + "}{", + "}{ foo", + "foo }{", + "a}{b", + "}}{{", + "} something {{{ {}}}d{ {}}}}{{{ }}}}}d{d something {{}}}}}}", + ] { + let result = QueryParser::new(GrepConfig).parse(query); + // A reversed-brace token must not be promoted to a Glob — there + // is no comma+letters between `{` and `}`, so it's just text. + assert!( + !result + .constraints + .iter() + .any(|c| matches!(c, Constraint::Glob(_))), + "GrepConfig: {query:?} produced a Glob constraint, got {:?}", + result.constraints + ); + + let result = QueryParser::new(crate::AiGrepConfig).parse(query); + assert!( + !result + .constraints + .iter() + .any(|c| matches!(c, Constraint::Glob(_))), + "AiGrepConfig: {query:?} produced a Glob constraint, got {:?}", + result.constraints + ); + } + } + + #[test] + fn test_grep_braces_without_comma_is_text() { + let parser = QueryParser::new(GrepConfig); + // Code patterns like format!("{}") should NOT be treated as brace expansion + let result = parser.parse(r#"format!("{}\\AppData", home)"#); + assert!( + result.constraints.is_empty(), + "Braces without comma should be text, got {:?}", + result.constraints + ); + assert_eq!(result.grep_text(), r#"format!("{}\\AppData", home)"#); + } + + #[test] + fn test_grep_valid_brace_expansion_amid_junk_braces() { + // A query mixing junk-brace tokens (`}{`, `{{}}`, `}}{{`, `{}`) with + // a real brace-expansion glob (`{src,lib}`) must NOT panic and MUST + // still surface the valid glob as a Glob constraint. Regression for + // the `}{` slice-out-of-bounds panic at config.rs:175. + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("}{ {{}} }}{{ {} {src,lib} pattern"); + + let glob_constraints: Vec<&str> = result + .constraints + .iter() + .filter_map(|c| match c { + Constraint::Glob(p) => Some(*p), + _ => None, + }) + .collect(); + assert_eq!( + glob_constraints, + vec!["{src,lib}"], + "Expected exactly one Glob({{src,lib}}), got {:?}", + result.constraints + ); + + // Same scenario for AiGrepConfig (delegates to GrepConfig::is_glob_pattern). + let parser = QueryParser::new(crate::AiGrepConfig); + let result = parser.parse("}{ {{}} }}{{ {} {src,lib} pattern"); + let glob_constraints: Vec<&str> = result + .constraints + .iter() + .filter_map(|c| match c { + Constraint::Glob(p) => Some(*p), + _ => None, + }) + .collect(); + assert_eq!( + glob_constraints, + vec!["{src,lib}"], + "AiGrepConfig: expected Glob({{src,lib}}), got {:?}", + result.constraints + ); + } + + #[test] + fn test_grep_format_braces_not_glob() { + let parser = QueryParser::new(GrepConfig); + // Code like format!("{}\\path", var) must not have tokens eaten as glob constraints. + // The trailing comma on the first token means both { } and , are present, + // but the comma is outside the braces so it should NOT trigger brace expansion. + let input = "format!(\"{}\\\\AppData\", home)"; + let result = parser.parse(input); + assert!( + result.constraints.is_empty(), + "format! pattern should have no constraints, got {:?}", + result.constraints + ); + } + + #[test] + fn test_grep_config_star_text_star_not_glob() { + use crate::GrepConfig; + let parser = QueryParser::new(GrepConfig); + let result = parser.parse("*quote* TODO"); + // Regular grep mode should NOT treat `*quote*` as a glob + assert!( + result.constraints.is_empty(), + "Expected no constraints in GrepConfig, got {:?}", + result.constraints + ); + } + + #[test] + fn test_file_picker_bare_filename_constraint() { + let parser = QueryParser::new(FilenameConstraintConfig); + let result = parser.parse("score.rs file_picker"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::FilePath("score.rs")), + "Expected FilePath(\"score.rs\"), got {:?}", + result.constraints[0] + ); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("file_picker")); + } + + #[test] + fn test_file_picker_path_prefixed_filename_constraint() { + let parser = QueryParser::new(FilenameConstraintConfig); + let result = parser.parse("libswscale/slice.c lum_convert"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!( + result.constraints[0], + Constraint::FilePath("libswscale/slice.c") + ), + "Expected FilePath(\"libswscale/slice.c\"), got {:?}", + result.constraints[0] + ); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("lum_convert")); + } + + #[test] + fn test_file_picker_single_token_filename_stays_fuzzy() { + let parser = QueryParser::new(FileSearchConfig); + // Single-token filename should NOT become a constraint -- it should + // return FFFQuery with Text fuzzy query so the caller uses it for fuzzy matching. + let result = parser.parse("score.rs"); + assert!(result.constraints.is_empty()); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("score.rs")); + } + + #[test] + fn test_absolute_path_with_location_not_path_segment() { + let parser = QueryParser::new(FileSearchConfig); + // Absolute file path with :line should parse as text + location, + // NOT as a PathSegment constraint (which would eat the whole token). + let result = parser.parse("/Users/neogoose/dev/fframes/src/renderer/concatenator.rs:12"); + assert!( + result.constraints.is_empty(), + "Absolute path with location should not become a constraint, got {:?}", + result.constraints + ); + assert_eq!( + result.fuzzy_query, + FuzzyQuery::Text("/Users/neogoose/dev/fframes/src/renderer/concatenator.rs") + ); + assert_eq!(result.location, Some(Location::Line(12))); + } + + #[test] + fn test_file_picker_filename_with_multiple_fuzzy_parts() { + let parser = QueryParser::new(FilenameConstraintConfig); + let result = parser.parse("main.rs src components"); + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::FilePath("main.rs") + )); + assert_eq!( + result.fuzzy_query, + FuzzyQuery::Parts(vec!["src", "components"]) + ); + } + + #[test] + fn test_file_picker_version_number_not_filename() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("v2.0 release"); + // v2.0 extension starts with digit → not a filename constraint + assert!( + result.constraints.is_empty(), + "v2.0 should not be a FilePath constraint, got {:?}", + result.constraints + ); + } + + #[test] + fn test_file_picker_only_one_filepath_constraint() { + let parser = QueryParser::new(FilenameConstraintConfig); + let result = parser.parse("main.rs score.rs"); + // Only first filename becomes a constraint; second is text + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::FilePath("main.rs") + )); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("score.rs")); + } + + #[test] + fn test_file_picker_filename_with_extension_constraint() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("main.rs *.lua"); + // With only path-scoping constraints (Extension) and no fuzzy text, + // `main.rs` is promoted to fuzzy text — the user is fuzzy-searching + // for "main.rs" among `.lua` files, not filtering by literal filename + // suffix. Only the Extension constraint remains. + assert_eq!(result.constraints.len(), 1); + assert!(matches!( + result.constraints[0], + Constraint::Extension("lua") + )); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("main.rs")); + } + + #[test] + fn test_file_picker_dotfile_is_filename() { + let parser = QueryParser::new(FilenameConstraintConfig); + let result = parser.parse(".gitignore src"); + assert_eq!(result.constraints.len(), 1); + assert!( + matches!(result.constraints[0], Constraint::FilePath(".gitignore")), + "Expected FilePath(\".gitignore\"), got {:?}", + result.constraints[0] + ); + assert_eq!(result.fuzzy_query, FuzzyQuery::Text("src")); + } + + #[test] + fn test_file_picker_no_extension_not_filename() { + let parser = QueryParser::new(FileSearchConfig); + let result = parser.parse("Makefile src"); + // No dot → not a filename constraint + assert!( + result.constraints.is_empty(), + "Makefile should not be a FilePath constraint, got {:?}", + result.constraints + ); + } +} diff --git a/doc/.gitkeep b/doc/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/doc/fff.nvim.txt b/doc/fff.nvim.txt new file mode 100644 index 0000000..a8e55b3 --- /dev/null +++ b/doc/fff.nvim.txt @@ -0,0 +1,451 @@ +*fff.nvim.txt* + For Neovim >= 0.10.0 Last change: 2026 June 30 + +============================================================================== +Table of Contents *fff.nvim-table-of-contents* + +1. fff.nvim |fff.nvim-fff.nvim| + +============================================================================== +1. fff.nvim *fff.nvim-fff.nvim* + +The best file search picker for Neovim. Frecency-ranked, typo-resistant, +git-award, very fast. + +Demo on the Linux kernel repo (100k files, 8GB): + + +https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb + + +INSTALLATION ~ + + +LAZY.NVIM + +>lua + { + 'dmtrKovalenko/fff.nvim', + build = function() + -- downloads a prebuilt binary or falls back to cargo build + require("fff.download").download_or_build_binary() + end, + -- for nixos: + -- build = "nix run .#release", + opts = { + debug = { + enabled = true, + show_scores = true, + }, + }, + lazy = false, -- the plugin lazy-initialises itself + keys = { + { "ff", function() require('fff').find_files() end, desc = 'FFFind files' }, + { "fg", function() require('fff').live_grep() end, desc = 'LiFFFe grep' }, + { "fz", + function() require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) end, + desc = 'Live fffuzy grep', + }, + { "fw", + function() require('fff').live_grep_under_cursor() end, + mode = { 'n', 'x' }, + desc = 'Search current word / selection', + }, + }, + } +< + + +VIM.PACK + +>lua + vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' }) + + vim.api.nvim_create_autocmd('PackChanged', { + callback = function(ev) + local name, kind = ev.data.spec.name, ev.data.kind + if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then + if not ev.data.active then vim.cmd.packadd('fff.nvim') end + require('fff.download').download_or_build_binary() + end + end, + }) + + vim.g.fff = { + lazy_sync = true, + debug = { enabled = true, show_scores = true }, + } + + vim.keymap.set('n', 'ff', function() require('fff').find_files() end, { desc = 'FFFind files' }) +< + + +PUBLIC API ~ + +>lua + require('fff').find_files() -- find files in current repo + require('fff').live_grep() -- live content grep + require('fff').live_grep_under_cursor() -- grep in normal, selection in visual + require('fff').scan_files() -- force rescan + require('fff').refresh_git_status() -- refresh git status + require('fff').find_files_in_dir(path) -- find in a specific dir + require('fff').change_indexing_directory(new_path) -- change root + + -- Programmatic search (no UI). Useful for plugin integrations. + require('fff').file_search(query, opts) -- fuzzy search files / dirs / mixed + require('fff').content_search(query, opts) -- programmatic grep +< + + +FILE_SEARCH(QUERY, OPTS) + +Returns a structured result `{ items, scores, total_matched, total_files?, +total_dirs?, location? }`. Each item has a `type` field (`"file"` or +`"directory"`) and `name` / `relative_path`. File items also expose `size`, +`modified`, `git_status`, `is_binary`, and frecency scores. + +>lua + local r = require('fff').file_search('button', { + mode = 'mixed', -- 'files' (default) | 'directories' | 'mixed' + max_results = 50, + page = 0, -- 0-based pagination + current_file = nil, -- path to deprioritize for distance scoring + max_threads = 4, + cwd = nil, -- switch indexed root if different (see below) + wait_for_index_ms = nil, -- override the default scan wait timeout + }) + for _, item in ipairs(r.items) do + print(item.type, item.relative_path) + end +< + + +CONTENT_SEARCH(QUERY, OPTS) + +Returns a `GrepResult` `{ items, total_matched, total_files_searched, +total_files, filtered_file_count, next_file_offset, regex_fallback_error? }`. +Each match item has `relative_path`, `name`, `line_number`, `col`, +`line_content`, `match_ranges`, plus the same file metadata as `file_search`. + +>lua + local r = require('fff').content_search('TODO', { + mode = 'plain', -- 'plain' (default) | 'regex' | 'fuzzy' + max_file_size = 10 * 1024 * 1024, + max_matches_per_file = 100, + smart_case = true, + page_size = 50, + file_offset = 0, + time_budget_ms = 0, + trim_whitespace = false, + cwd = nil, -- switch indexed root if different + wait_for_index_ms = nil, -- override the default scan wait timeout + }) + for _, m in ipairs(r.items) do + print(string.format('%s:%d %s', m.relative_path, m.line_number, m.line_content)) + end +< + +Both functions accept the same constraint syntax as the UI pickers (e.g. +`git:modified`, `*.rs`, `!test/`, glob patterns). + + +CWD AND INDEXING + +Both `file_search` and `content_search` honour an optional `cwd` field. The +first call to either function lazily initialises the picker at +`config.base_path` (your Neovim cwd by default). + +- If `cwd` matches the currently indexed root, the call returns immediately against the existing index. +- If `cwd` differs, the picker is re-indexed at the new root and the call **blocks** (default up to 10 s) until the new picker is installed and its initial scan completes — so callers always get results from the right tree. +- If the index is still warming up after a `change_indexing_directory`, you can pass `wait_for_index_ms = N` to block for up to `N` ms regardless of whether `cwd` triggered the swap. Pass `0` to skip waiting entirely (useful for fire-and-forget calls where partial results are acceptable). +- Invalid or non-existent `cwd` paths return an empty result and emit an error via `vim.notify`. + + +COMMANDS ~ + +- `:FFFScan`. Rescan files. +- `:FFFRefreshGit`. Refresh git status. +- `:FFFClearCache [all|frecency|files]`. Clear caches. +- `:FFFHealth`. Health check. +- `:FFFDebug [on|off|toggle]`. Toggle the scoring display. +- `:FFFOpenLog`. Open `~/.local/state/nvim/log/fff.log`. + + +CONFIGURATION ~ + +Defaults are sensible. Override only what you care about. + +>lua + require('fff').setup({ + base_path = vim.fn.getcwd(), + prompt = '> ', + title = 'FFFiles', + max_results = 100, + max_threads = 4, + lazy_sync = true, + prompt_vim_mode = false, + follow_symlinks = false, + -- Allow indexing the user's $HOME directory. Enabled by default. + -- Disable if you strictly sure you don't want this, as it makes whole fff error hard + enable_home_dir_scanning = true, + -- Allow indexing a filesystem root (e.g. `/`, `C:\`). Disabled by default + enable_fs_root_scanning = false, + layout = { + height = 0.8, + width = 0.8, + prompt_position = 'bottom', -- or 'top' + preview_position = 'right', -- 'left' | 'right' | 'top' | 'bottom' + preview_size = 0.5, + -- Border style for the picker windows. Leave unset (nil) to follow the + -- global `vim.o.winborder`; set it to override fff's borders independently. + border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none' + flex = { size = 130, wrap = 'top' }, + min_list_height = 10, -- do not display anything except the list below this threshold + show_scrollbar = true, + path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end' | 'start' + anchor = 'center', + }, + preview = { + enabled = true, + max_size = 10 * 1024 * 1024, + chunk_size = 8192, + binary_file_threshold = 1024, + imagemagick_info_format_str = '%m: %wx%h, %[colorspace], %q-bit', + line_numbers = false, + cursorlineopt = 'both', + wrap_lines = false, + filetypes = { + svg = { wrap_lines = true }, + markdown = { wrap_lines = true }, + text = { wrap_lines = true }, + }, + }, + keymaps = { + close = '', + select = '', + select_split = '', + select_vsplit = '', + select_tab = '', + move_up = { '', '' }, + move_down = { '', '' }, + preview_scroll_up = '', + preview_scroll_down = '', + toggle_debug = '', + cycle_grep_modes = '', + -- grep mode only: jump cursor to first match of next/prev file group + grep_jump_to_next_file = { '', '' }, + grep_jump_to_prev_file = { '', '' }, + cycle_previous_query = '', + toggle_select = '', + send_to_quickfix = '', + focus_list = 'l', + focus_preview = 'p', + }, + frecency = { + enabled = true, + db_path = vim.fn.stdpath('cache') .. '/fff_nvim', + }, + history = { + enabled = true, + db_path = vim.fn.stdpath('data') .. '/fff_queries', + min_combo_count = 3, + combo_boost_score_multiplier = 100, + }, + git = { + status_text_color = false, -- true to color filenames by git status + }, + select = { + -- Return winid to open the chosen file in, or nil to open in the original window + select_window = function(current_buf, action) --[[ default impl ]] end, + }, + grep = { + max_file_size = 10 * 1024 * 1024, + max_matches_per_file = 100, + smart_case = true, + time_budget_ms = 150, + modes = { 'plain', 'regex', 'fuzzy' }, + trim_whitespace = false, + enable_filename_constraint = false, -- treat filename-like tokens (e.g. `score.rs`) in a grep query as a file-path filter scoping the search; off = searched as literal text + location_format = ':%d:%d', -- printf format for line:col prefix in grep results, e.g. ':%d' for line-only + }, + debug = { + enabled = false, -- show the file info panel next to the preview + show_scores = false, -- inline scores in the file list + -- Per-section toggles for the file info panel. Accepts a boolean shorthand + -- (`show_file_info = true|false`) to flip everything at once. The panel + -- adapts to width: narrow renders sections vertically, wide renders them + -- as a two-column grid. Disable a section to also shrink the panel. + show_file_info = { + file_info = true, -- size, type, git status, frecency + score_breakdown = true, -- total + match type, bonuses, modifiers, penalty + -- modified + accessed timestamps; pass a table to hide individual rows: + -- timings = { modified = false, accessed = true } + timings = true, + full_path = true, -- relative path at the bottom (wraps if too long) + }, + }, + logging = { + -- logs will be written in a parent directory of this file path in files like + -- `++.`. Run :FFFOpenLog to open current one + log_file = vim.fn.stdpath('log') .. '/fff.log', + log_level = 'info', + retain_runs = 20, + }, + }) +< + + +LIVE GREP MODES ~ + +`` cycles between `plain`, `regex`, and `fuzzy`. The list is +configurable via `grep.modes`, and single-mode setups hide the indicator +entirely. + +Per-call override: + +>lua + require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) + require('fff').live_grep({ query = 'search term' }) -- pre-fill +< + + +CONSTRAINTS ~ + +Both find and grep accept these tokens to refine a query: + +- `git:modified`. One of `modified`, `staged`, `deleted`, `renamed`, `untracked`, `ignored`. +- `test/`. Any deeply nested children of `test/`. +- `!something`, `!test/`, `!git:modified`. Exclusion. +- `./**/*.{rs,lua}`. Any valid glob, powered by zlob . + +Grep-only: + +- `*.md`, `*.{c,h}`. Extension filter. +- `src/main.rs`. Grep inside a single file. + +Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`. + + +OPEN IN INVOKING WINDOW ~ + +By default fff.nvim will try to open a file in the most suitable window, so any +non-file buffers are not affected. You can customize or disable this by +providing: + +>lua + require('fff').setup({ + select = { + select_window = function(_current_buf, _action) return nil end, + }, + }) +< + +Caveat: the chosen file replaces the buffer in the invoking window even if +it’s a non-modifiable / special buftype. `winfixbuf` windows still fall back +to `:split` to avoid `E1513`. + + +MULTI-SELECT AND QUICKFIX ~ + +- ``. Toggle selection (shows a thick `▊` in the signcolumn). +- ``. Send selected files to the quickfix list and close the picker. + + +GIT STATUS HIGHLIGHTING ~ + +Sign-column indicators are on by default. To color filename text by git status, +set `git.status_text_color = true` and adjust the `hl.git_*` groups. See `:help +fff.nvim` for the full list. + + +FLOAT COLORS ~ + +The picker maps its float content to `NormalFloat` (via `hl.normal`) and the +border to `FloatBorder`. Default `FloatBorder` links to `NormalFloat`, so +border and content share a background out of the box and the picker reads as a +single popup. Override `hl.normal = 'Normal'` to make the picker blend with the +editor instead. + +For finer control, set `hl.winhl` to override the per-window `winhighlight`. It +accepts either a single string applied to every picker window, or a table with +optional `prompt`, `list`, `preview`, and `file_info` keys. Missing keys fall +back to the default built from `hl.normal`, `hl.border`, and `hl.title`. + +>lua + -- Apply the same winhighlight to all picker windows + hl = { winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title' } + + -- Or override specific windows only + hl = { + winhl = { + prompt = 'Normal:Pmenu,FloatBorder:FloatBorder', + list = 'Normal:NormalFloat,FloatBorder:FloatBorder', + preview = 'Normal:NormalFloat,FloatBorder:FloatBorder', + }, + } +< + + +FILE INFO PANEL ~ + +Enable with `debug.enabled = true`. The panel sits above the preview and shows +file metadata, score breakdown, timestamps and the full absolute path. It +adapts to the panel width: at narrow widths sections stack vertically (B2), at +wide widths sections render as a two-column grid (H2). Each section can be +disabled individually via `debug.show_file_info`. + +Customise the panel via `hl`: + + -------------------------------------------------------------------------- + key default used for + ----------------------- ----------------- -------------------------------- + file_info_section Title section header label + + file_info_separator FloatBorder dashes that act as section + borders + + file_info_label Comment row labels (Size, Type, Git, …) + + file_info_value Normal fg plain values + + file_info_value_dim NonText dim values, separators inside + rows + + file_info_size Number file size value + + file_info_type Type filetype value + + file_info_path Directory full path + + file_info_total_score bold + Number total score (bold) + + file_info_match_type bold + Special match type (bold) + + file_info_score_pos DiagnosticOk positive score components + + file_info_score_neg DiagnosticError negative score components + -------------------------------------------------------------------------- + +FILE FILTERING ~ + +FFF honours `.gitignore`. For picker-only ignores that do not touch git, add a +sibling `.ignore` file: + +>gitignore + *.md + docs/archive/**/*.md +< + +Run `:FFFScan` to force a rescan. + + +TROUBLESHOOTING ~ + +- `:FFFHealth` verifies picker init, optional dependencies, and DB connectivity. +- `:FFFOpenLog` opens the current session’s log file. +- Historical log files are stored near the main log file `/log/fff++.log` (up to 20 files) +- For a crash backtrace, run `lldb -- nvim` or `gdb -- nvim` and reproduce + +Generated by panvimdoc + +vim:tw=78:ts=8:noet:ft=help:norl: diff --git a/empty_config.lua b/empty_config.lua new file mode 100644 index 0000000..8d23d9b --- /dev/null +++ b/empty_config.lua @@ -0,0 +1,62 @@ +-- Single file Neovim config for testing fff.nvim locally +-- Usage: nvim -u /Users/neogoose/dev/fff.nvim/init.lua + +-- Set up lazy.nvim plugin manager +local lazypath = vim.fn.stdpath('data') .. '/lazy/lazy.nvim' +if not vim.loop.fs_stat(lazypath) then + vim.fn.system({ + 'git', + 'clone', + '--filter=blob:none', + 'https://github.com/folke/lazy.nvim.git', + '--branch=stable', + lazypath, + }) +end +vim.opt.rtp:prepend(lazypath) + +require('lazy').setup({ + { + dir = '~/dev/fff.nvim', + 'https://github.com/dmtrKovalenko/fff.nvim', + build = function() + -- this will download prebuild binary or try to use existing rustup toolchain to build from source + -- (if you are using lazy you can use gb for rebuilding a plugin if needed) + require('fff.download').download_or_build_binary() + end, + dependencies = { + 'nvim-tree/nvim-web-devicons', -- Optional: for file icons + -- { + -- 'nvim-mini/mini.icons', + -- version = false, + -- config = true, + -- }, + }, + config = function() + require('fff').setup({ + -- Configure fff.nvim here + ui = { + width = 0.8, + height = 0.8, + }, + file_picker = { + auto_reload_on_write = true, + frecency_boost = true, + }, + }) + end, + }, +}, { + root = vim.fn.stdpath('data') .. '/fff-empty-test', + lockfile = vim.fn.stdpath('data') .. '/fff-empty-test.json', +}) + +vim.opt.number = true +vim.opt.relativenumber = true + +vim.keymap.set('n', 'ff', function() require('fff').find_files() end, { desc = 'Find files' }) +vim.keymap.set('n', 'fg', function() require('fff').find_in_git_root() end, { desc = 'Find files in git root' }) +vim.keymap.set('n', 'fr', function() require('fff').scan_files() end, { desc = 'Rescan files' }) +vim.keymap.set('n', 'fs', function() require('fff').refresh_git_status() end, { desc = 'Refresh git status' }) + +vim.notify('FFF.nvim local config loaded! Press ff', vim.log.levels.INFO) diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..12c3c1a --- /dev/null +++ b/flake.lock @@ -0,0 +1,153 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1773857772, + "narHash": "sha256-5xsK26KRHf0WytBtsBnQYC/lTWDhQuT57HJ7SzuqZcM=", + "owner": "ipetkov", + "repo": "crane", + "rev": "b556d7bbae5ff86e378451511873dfd07e4504cd", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "zig-overlay": "zig-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1773803479, + "narHash": "sha256-GD6i1F2vrSxbsmbS92+8+x3DbHOJ+yrS78Pm4xigW4M=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "f17186f52e82ec5cf40920b58eac63b78692ac7c", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "flake": false, + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "zig-overlay": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems_2" + }, + "locked": { + "lastModified": 1776789209, + "narHash": "sha256-G6B7Q4TXn7MZ1mB+f9rymjsYF5PLWoSvmbxijb/99bw=", + "owner": "mitchellh", + "repo": "zig-overlay", + "rev": "14fe971844e841297ddd2ce9783d6892b467af39", + "type": "github" + }, + "original": { + "owner": "mitchellh", + "repo": "zig-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..6a079f1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,154 @@ +{ + description = "fff.nvim"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + crane.url = "github:ipetkov/crane"; + + flake-utils.url = "github:numtide/flake-utils"; + + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + zig-overlay = { + url = "github:mitchellh/zig-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + self, + nixpkgs, + crane, + flake-utils, + rust-overlay, + zig-overlay, + ... + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + + # zlob requires Zig >= 0.16, but nixpkgs tops out at 0.15. Pull from + # mitchellh/zig-overlay which ships every released version. + zig = zig-overlay.packages.${system}."0.16.0"; + + rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + + craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; + + cargoToml = builtins.fromTOML (builtins.readFile ./crates/fff-nvim/Cargo.toml); + fffMcpCargoToml = builtins.fromTOML (builtins.readFile ./crates/fff-mcp/Cargo.toml); + + # Common arguments can be set here to avoid repeating them later + # Note: changes here will rebuild all dependency crates + commonArgs = { + pname = cargoToml.package.name; + version = cargoToml.package.version; + src = craneLib.cleanCargoSource ./.; + strictDeps = true; + + nativeBuildInputs = [ pkgs.pkg-config pkgs.perl zig pkgs.llvmPackages.libclang.lib ]; + buildInputs = with pkgs; [ + # Add additional build inputs here + openssl + ]; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + + # Zig 0.16 insists on writing to its global cache even when the + # zlob build.rs passes --global-cache-dir. In the nix sandbox $HOME + # is /homeless-shelter (unwritable), so redirect to $TMPDIR before + # the build phase runs. + preBuild = '' + export ZIG_GLOBAL_CACHE_DIR="$TMPDIR/zig-global-cache" + export ZIG_LOCAL_CACHE_DIR="$TMPDIR/zig-local-cache" + export XDG_CACHE_HOME="$TMPDIR/cache" + mkdir -p "$ZIG_GLOBAL_CACHE_DIR" "$ZIG_LOCAL_CACHE_DIR" "$XDG_CACHE_HOME" + ''; + }; + + my-crate = craneLib.buildPackage ( + commonArgs + // { + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + doCheck = false; + } + ); + + fff-mcp-args = commonArgs // { + pname = fffMcpCargoToml.package.name; + version = fffMcpCargoToml.package.version; + cargoExtraArgs = "-p fff-mcp --bin fff-mcp"; + }; + + fff-mcp = craneLib.buildPackage ( + fff-mcp-args + // { + cargoArtifacts = craneLib.buildDepsOnly fff-mcp-args; + doCheck = false; + } + ); + + # Copies the dynamic library into the target/release folder + copy-dynamic-library = /* bash */ '' + set -eo pipefail + mkdir -p target/release + if [ "$(uname)" = "Darwin" ]; then + cp -vf ${my-crate}/lib/libfff_nvim.dylib target/release/libfff_nvim.dylib + else + cp -vf ${my-crate}/lib/libfff_nvim.so target/release/libfff_nvim.so + fi + echo "Library copied to target/release/" + ''; + in + { + checks = { + inherit my-crate fff-mcp; + }; + + packages = { + default = my-crate; + inherit fff-mcp; + + # Neovim plugin + fff-nvim = pkgs.vimUtils.buildVimPlugin { + pname = "fff.nvim"; + version = "main"; + src = pkgs.lib.cleanSource ./.; + postPatch = copy-dynamic-library; + doCheck = false; # Skip require check since we have a Rust FFI component + }; + }; + + apps.default = flake-utils.lib.mkApp { + drv = my-crate; + }; + + apps.fff-mcp = flake-utils.lib.mkApp { + drv = fff-mcp; + }; + + # Add the release command + apps.release = flake-utils.lib.mkApp { + drv = pkgs.writeShellScriptBin "release" copy-dynamic-library; + }; + + devShells.default = craneLib.devShell { + # Inherit inputs from checks. + checks = self.checks.${system}; + # Extra inputs can be added here; cargo and rustc are provided by default. + packages = [ + # pkgs.ripgrep + ]; + }; + } + ); +} diff --git a/install-mcp.ps1 b/install-mcp.ps1 new file mode 100644 index 0000000..dfa1090 --- /dev/null +++ b/install-mcp.ps1 @@ -0,0 +1,261 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + FFF MCP Server installer for Windows. +.DESCRIPTION + Pipe usage: + irm https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.ps1 | iex + Direct usage (supports params): + iwr https://.../install-mcp.ps1 -OutFile install-mcp.ps1; .\install-mcp.ps1 -Version v0.1.2 + Env-var fallbacks (for the piped form): + $env:FFF_MCP_VERSION, $env:FFF_MCP_INSTALL_DIR +.PARAMETER Version + Release tag to install (e.g. 'v0.1.2'). Default: latest release containing a Windows MCP asset. +.PARAMETER InstallDir + Target install directory. Default: $env:LOCALAPPDATA\fff-mcp\bin. +.PARAMETER PathScope + How to persist PATH: 'User' (set user env var, default), 'Profile' (append to $PROFILE *nix-style), 'None' (do not persist). + Env-var fallback: $env:FFF_MCP_PATH_SCOPE. +#> +param( + [string]$Version = $env:FFF_MCP_VERSION, + [string]$InstallDir = $env:FFF_MCP_INSTALL_DIR, + [ValidateSet('User', 'Profile', 'None')] + [string]$PathScope = $(if ($env:FFF_MCP_PATH_SCOPE) { $env:FFF_MCP_PATH_SCOPE } else { 'User' }) +) + +$ErrorActionPreference = 'Stop' + +# Force TLS 1.2 — PS 5.1 on older Win10 may default to SSL3/TLS1.0 which GitHub rejects. +[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + +$Repo = 'dmtrKovalenko/fff.nvim' +$BinaryName = 'fff-mcp' +if (-not $InstallDir) { $InstallDir = Join-Path $env:LOCALAPPDATA 'fff-mcp\bin' } + +function Write-Info { param($m) Write-Host $m -ForegroundColor Blue } +function Write-Success { param($m) Write-Host $m -ForegroundColor DarkYellow } +function Write-Warn { param($m) Write-Host $m -ForegroundColor Yellow } + +function Get-Target { + # Read from registry — env vars lie under x86/ARM64 emulation. Same approach Bun uses. + $arch = (Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment').PROCESSOR_ARCHITECTURE + switch ($arch) { + 'AMD64' { return 'x86_64-pc-windows-msvc' } + 'ARM64' { return 'aarch64-pc-windows-msvc' } + default { throw "Unsupported architecture: $arch" } + } +} + +function Get-LatestReleaseTag { + param([string]$Target) + $asset = "$BinaryName-$Target.exe" + $headers = @{ 'User-Agent' = 'fff-mcp-installer' } + if ($env:GITHUB_TOKEN) { $headers['Authorization'] = "Bearer $env:GITHUB_TOKEN" } + + $releases = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repo/releases" -Headers $headers + $rel = $releases | Where-Object { $_.assets.name -contains $asset } | Select-Object -First 1 + if (-not $rel) { + throw "No release found containing $asset. The MCP build may not have been released for this platform yet." + } + return $rel.tag_name +} + +function Invoke-Download { + param([string]$Url, [string]$OutFile) + # curl.exe (ships with Win10 1803+) is faster than iwr on PS 5.1. Fall back to iwr. + $curl = Get-Command curl.exe -ErrorAction SilentlyContinue + if ($curl) { + & $curl.Source -fsSL -o $OutFile $Url + if ($LASTEXITCODE -ne 0) { throw "curl.exe exited with $LASTEXITCODE" } + } else { + $prev = $ProgressPreference + try { + # iwr progress bar tanks throughput on PS 5.1. + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri $Url -OutFile $OutFile -UseBasicParsing + } finally { + $ProgressPreference = $prev + } + } +} + +function Install-Binary { + param([string]$Target, [string]$Tag) + + $filename = "$BinaryName-$Target.exe" + $url = "https://github.com/$Repo/releases/download/$Tag/$filename" + + Write-Info "Downloading $filename from release $Tag..." + + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) + New-Item -ItemType Directory -Force -Path $tmp | Out-Null + try { + $tmpFile = Join-Path $tmp $filename + try { + Invoke-Download -Url $url -OutFile $tmpFile + } catch { + Write-Host "" + Write-Host "Error: Failed to download binary for your platform." -ForegroundColor Red + Write-Host " URL: $url" + Write-Host " Release: $Tag" + Write-Host " Platform: $Target" + Write-Host "Check available releases at: https://github.com/$Repo/releases" + throw + } + + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + $dest = Join-Path $InstallDir "$BinaryName.exe" + Move-Item -Force -Path $tmpFile -Destination $dest + return $dest + } finally { + Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue + } +} + +function Test-OnPath { + param([string]$Dir) + $paths = $env:PATH -split ';' + return ($paths -contains $Dir) -or ($paths -contains $Dir.TrimEnd('\')) +} + +function Add-ToUserPath { + param([string]$Dir) + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + if (-not $userPath) { $userPath = '' } + $entries = $userPath -split ';' | Where-Object { $_ -ne '' } + if ($entries -notcontains $Dir) { + $newPath = (@($entries + $Dir) -join ';') + [Environment]::SetEnvironmentVariable('Path', $newPath, 'User') + Write-Success "Added $Dir to user PATH." + } +} + +function Add-ToProfilePath { + param([string]$Dir) + $profilePath = $PROFILE.CurrentUserAllHosts + $line = "`$env:PATH += ';$Dir' # added by fff-mcp installer" + if (Test-Path $profilePath) { + $existing = Get-Content $profilePath -Raw -ErrorAction SilentlyContinue + if ($existing -and $existing.Contains($Dir)) { return } + } else { + New-Item -ItemType File -Force -Path $profilePath | Out-Null + } + Add-Content -Path $profilePath -Value "`n$line" + Write-Success "Appended PATH update to $profilePath." +} + +function Set-Path { + param([string]$Dir, [string]$Scope) + switch ($Scope) { + 'User' { Add-ToUserPath $Dir } + 'Profile' { Add-ToProfilePath $Dir } + 'None' { Write-Info "Skipping PATH persistence (-PathScope None)." } + } + # Make available in current session regardless of scope. + if (-not (Test-OnPath $Dir)) { $env:PATH = "$env:PATH;$Dir" } +} + +function Show-SetupInstructions { + param([string]$BinaryPath) + $foundAny = $false + + Write-Host "" + Write-Success "FFF MCP Server installed successfully!" + Write-Host "" + Write-Info "Setup with your AI coding assistant:" + Write-Host "" + + if (Get-Command claude -ErrorAction SilentlyContinue) { + $foundAny = $true + Write-Success "[Claude Code] detected" + Write-Host "" + Write-Host "Global (recommended):" + Write-Host "claude mcp add -s user fff -- $BinaryPath" + Write-Host "" + Write-Host "Or project-level .mcp.json (uses PATH):" + Write-Host @' +{ + "mcpServers": { + "fff": { + "type": "stdio", + "command": "fff-mcp", + "args": [] + } + } +} +'@ + Write-Host "" + } + + if (Get-Command opencode -ErrorAction SilentlyContinue) { + $foundAny = $true + Write-Success "[OpenCode] detected" + Write-Host "Add to your opencode.json:" + Write-Host @' +{ + "mcp": { + "fff": { + "type": "local", + "command": ["fff-mcp"], + "enabled": true + } + } +} +'@ + Write-Host "" + } + + if (Get-Command codex -ErrorAction SilentlyContinue) { + $foundAny = $true + Write-Success "[Codex] detected" + Write-Host "codex mcp add fff -- fff-mcp" + Write-Host "" + } + + if (-not $foundAny) { + Write-Host "No AI coding assistants detected." + Write-Host "Binary path: $BinaryPath" + Write-Host "" + } + + Write-Host "Binary: $BinaryPath" + Write-Host "Docs: https://github.com/$Repo" + Write-Host "" + Write-Info "Tip: Add this to your CLAUDE.md or AGENTS.md to make AI use fff for all searches:" + Write-Host '"Use the fff MCP tools for all file search operations instead of default tools."' +} + +function Main { + $target = Get-Target + + $existing = Join-Path $InstallDir "$BinaryName.exe" + $isUpdate = Test-Path $existing + + if ($isUpdate) { + Write-Info "Updating FFF MCP Server..." + } else { + Write-Info "Installing FFF MCP Server..." + } + Write-Host "" + Write-Info "Detected platform: $target" + + if ($Version) { + $tag = $Version + Write-Info "Using pinned version: $tag" + } else { + $tag = Get-LatestReleaseTag -Target $target + } + $binaryPath = Install-Binary -Target $target -Tag $tag + + if ($isUpdate) { + Write-Host "" + Write-Success "FFF MCP Server updated to $tag!" + Write-Host "" + } else { + Set-Path -Dir $InstallDir -Scope $PathScope + Show-SetupInstructions -BinaryPath $binaryPath + } +} + +Main diff --git a/install-mcp.sh b/install-mcp.sh new file mode 100755 index 0000000..aae6216 --- /dev/null +++ b/install-mcp.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +set -eo pipefail + +# FFF MCP Server installer +# Usage: curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash + +REPO="dmtrKovalenko/fff.nvim" +BINARY_NAME="fff-mcp" +INSTALL_DIR="${FFF_MCP_INSTALL_DIR:-$HOME/.local/bin}" + +PINNED_RELEASE_TAG="v0.9.6" + +SHA256_X86_64_UNKNOWN_LINUX_MUSL="102ceaf173ef776becb3322216e9f6b5caef997c400c5d15f112ce4de40a1f5a" +SHA256_AARCH64_UNKNOWN_LINUX_MUSL="a9810c9056afa6d9e8ac1e7a3f1f15f8ecbfdc16a592e9d26e4b2434ef97a675" +SHA256_X86_64_APPLE_DARWIN="58259324c2c13a1b6f24f13138c2cd3eae9ff20e05201a539beb8f2044a651aa" +SHA256_AARCH64_APPLE_DARWIN="29a7fadeafb062f3e5954b1ab8c69e14dca24f5e061cd8d3b1ea1bab385a3754" +SHA256_X86_64_PC_WINDOWS_MSVC="7ff688d034aa42ff779a61ad12689794bdc253c895152796046f374390fb9cad" +SHA256_AARCH64_PC_WINDOWS_MSVC="94ce316a38775d8ed3b32882d8ef45560740319de8d7ac7bb80c31887b3afb6d" + +expected_sha_for() { + case "$1" in + x86_64-unknown-linux-musl) echo "$SHA256_X86_64_UNKNOWN_LINUX_MUSL" ;; + aarch64-unknown-linux-musl) echo "$SHA256_AARCH64_UNKNOWN_LINUX_MUSL" ;; + x86_64-apple-darwin) echo "$SHA256_X86_64_APPLE_DARWIN" ;; + aarch64-apple-darwin) echo "$SHA256_AARCH64_APPLE_DARWIN" ;; + x86_64-pc-windows-msvc) echo "$SHA256_X86_64_PC_WINDOWS_MSVC" ;; + aarch64-pc-windows-msvc) echo "$SHA256_AARCH64_PC_WINDOWS_MSVC" ;; + esac +} + +info() { printf '\033[1;34m%s\033[0m\n' "$*"; } +success() { printf '\033[1;38;5;208m%s\033[0m\n' "$*"; } +warn() { printf '\033[1;33m%s\033[0m\n' "$*"; } +error() { printf '\033[1;31mError: %s\033[0m\n' "$*" >&2; exit 1; } + +# Print JSON with syntax highlighting via jq if available, plain otherwise +print_json() { + if command -v jq &>/dev/null; then + echo "$1" | jq . + else + echo "$1" + fi +} + +detect_platform() { + local os arch target + + os="$(uname -s)" + arch="$(uname -m)" + + case "$os" in + Linux) + # Prefer musl (static) for maximum compatibility + case "$arch" in + x86_64) target="x86_64-unknown-linux-musl" ;; + aarch64|arm64) target="aarch64-unknown-linux-musl" ;; + *) error "Unsupported architecture: $arch" ;; + esac + ;; + Darwin) + case "$arch" in + x86_64) target="x86_64-apple-darwin" ;; + aarch64|arm64) target="aarch64-apple-darwin" ;; + *) error "Unsupported architecture: $arch" ;; + esac + ;; + MINGW*|MSYS*|CYGWIN*) + case "$arch" in + x86_64) target="x86_64-pc-windows-msvc" ;; + aarch64|arm64) target="aarch64-pc-windows-msvc" ;; + *) error "Unsupported architecture: $arch" ;; + esac + ;; + *) error "Unsupported OS: $os" ;; + esac + + echo "$target" +} + +get_latest_release_tag() { + local target="$1" + + # Honor the pin baked in by `make bump-install-mcp-sh`. Required when SHAs + # are pinned, since fetching /releases would race against newer releases. + if [ -n "$PINNED_RELEASE_TAG" ]; then + echo "$PINNED_RELEASE_TAG" + return + fi + + local releases_json + local curl_args=(-fsSL) + + # Use gh CLI token if available to avoid rate limiting + if command -v gh &>/dev/null; then + local gh_token + gh_token="$(gh auth token 2>/dev/null || true)" + if [ -n "$gh_token" ]; then + curl_args+=(-H "Authorization: token $gh_token") + fi + fi + + releases_json=$(curl "${curl_args[@]}" "https://api.github.com/repos/${REPO}/releases") \ + || error "Failed to fetch releases from https://github.com/${REPO}/releases" + + # Find the first release that contains an fff-mcp binary for our platform + local tag + tag=$(echo "$releases_json" \ + | grep -oE '"(tag_name|name)": *"[^"]*"' \ + | awk -v target="fff-mcp-${target}" ' + /"tag_name":/ { gsub(/.*": *"|"/, ""); current_tag = $0; next } + /"name":/ && index($0, target) { print current_tag; exit } + ') + + if [ -z "$tag" ]; then + error "No release found containing fff-mcp binaries for ${target}. The MCP build may not have been released yet." + fi + echo "$tag" +} + +download_binary() { + local target="$1" + local tag="$2" + local ext="" + + case "$target" in + *windows*) ext=".exe" ;; + esac + + local filename="${BINARY_NAME}-${target}${ext}" + local url="https://github.com/${REPO}/releases/download/${tag}/${filename}" + local checksum_url="${url}.sha256" + + info "Downloading ${filename} from release ${tag}..." + + local tmp_dir + tmp_dir="$(mktemp -d)" + trap 'rm -rf "$tmp_dir"' EXIT + + if ! curl -fsSL -o "${tmp_dir}/${filename}" "$url" 2>/dev/null; then + echo "" >&2 + printf '\033[1;31mError: Failed to download binary for your platform.\033[0m\n' >&2 + echo "" >&2 + echo " URL: ${url}" >&2 + echo " Release: ${tag}" >&2 + echo " Platform: ${target}" >&2 + echo "" >&2 + echo "This likely means the MCP binary hasn't been built for this release yet." >&2 + echo "Check available releases at: https://github.com/${REPO}/releases" >&2 + exit 1 + fi + + # Verify checksum: prefer the SHA pinned in this script (offline, tamper-evident). + # Fall back to the .sha256 file on the release for targets/releases without a pin. + if command -v sha256sum &>/dev/null; then + local pinned_sha + pinned_sha="$(expected_sha_for "$target")" + if [ -n "$pinned_sha" ]; then + info "Verifying checksum against pinned value..." + echo "${pinned_sha} ${filename}" > "${tmp_dir}/${filename}.sha256" + (cd "$tmp_dir" && sha256sum -c "${filename}.sha256") \ + || error "Checksum verification failed!" + elif curl -fsSL -o "${tmp_dir}/${filename}.sha256" "$checksum_url" 2>/dev/null; then + info "Verifying checksum..." + (cd "$tmp_dir" && sha256sum -c "${filename}.sha256") \ + || error "Checksum verification failed!" + else + warn "Checksum file not available, skipping verification." + fi + fi + + # Install + mkdir -p "$INSTALL_DIR" + mv "${tmp_dir}/${filename}" "${INSTALL_DIR}/${BINARY_NAME}${ext}" + chmod +x "${INSTALL_DIR}/${BINARY_NAME}${ext}" + + if [ "$IS_UPDATE" != true ]; then + success "Installed ${BINARY_NAME} to ${INSTALL_DIR}/${BINARY_NAME}${ext}" + fi +} + +check_path() { + case ":$PATH:" in + *":${INSTALL_DIR}:"*) return 0 ;; + esac + + warn "${INSTALL_DIR} is not in your PATH." + echo "" + echo "Add it to your shell profile:" + echo "" + + local shell_name + shell_name="$(basename "${SHELL:-bash}")" + case "$shell_name" in + zsh) + echo " echo 'export PATH=\"${INSTALL_DIR}:\$PATH\"' >> ~/.zshrc" + echo " source ~/.zshrc" + ;; + fish) + echo " fish_add_path ${INSTALL_DIR}" + ;; + *) + echo " echo 'export PATH=\"${INSTALL_DIR}:\$PATH\"' >> ~/.bashrc" + echo " source ~/.bashrc" + ;; + esac + echo "" +} + +print_setup_instructions() { + local binary_path="${INSTALL_DIR}/${BINARY_NAME}" + local found_any=false + + echo "" + success "FFF MCP Server installed successfully!" + echo "" + info "Setup with your AI coding assistant:" + echo "" + + # Claude Code + if command -v claude &>/dev/null; then + found_any=true + success "[Claude Code] detected" + echo "" + echo "Global (recommended):" + echo "claude mcp add -s user fff -- ${binary_path}" + echo "" + echo "Or project-level .mcp.json (uses PATH):" + echo "" + print_json '{ + "mcpServers": { + "fff": { + "type": "stdio", + "command": "fff-mcp", + "args": [] + } + } +}' + echo "" + fi + + # OpenCode + if command -v opencode &>/dev/null; then + found_any=true + success "[OpenCode] detected" + echo "" + echo "Add to ~/.config/opencode/opencode.json:" + echo "" + print_json '{ + "mcp": { + "fff": { + "type": "local", + "command": ["fff-mcp"], + "enabled": true + } + } +}' + echo "" + fi + + # Codex + if command -v codex &>/dev/null; then + found_any=true + success "[Codex] detected" + echo "" + echo "codex mcp add fff -- fff-mcp" + echo "" + fi + + if [ "$found_any" = false ]; then + echo "No AI coding assistants detected." + echo "" + echo "Binary path: ${binary_path}" + echo "" + fi + + echo "Binary: ${binary_path}" + echo "Docs: https://github.com/${REPO}" + echo "" + info "Tip: Add this to your CLAUDE.md or AGENTS.md to make AI use fff for all searches:" + echo "\"" + echo "Use the fff MCP tools for all file search operations instead of default tools." + echo "\"" + + +} + +main() { + local target + target="$(detect_platform)" + + local existing_binary="${INSTALL_DIR}/${BINARY_NAME}" + IS_UPDATE=false + + if [ -x "$existing_binary" ]; then + IS_UPDATE=true + info "Updating FFF MCP Server..." + else + info "Installing FFF MCP Server..." + fi + echo "" + + info "Detected platform: ${target}" + + local tag + tag="$(get_latest_release_tag "$target")" + + download_binary "$target" "$tag" + + if [ "$IS_UPDATE" = true ]; then + echo "" + success "FFF MCP Server updated to ${tag}!" + echo "" + else + check_path + print_setup_instructions + fi +} + +main diff --git a/lua/fff.lua b/lua/fff.lua new file mode 100644 index 0000000..886502c --- /dev/null +++ b/lua/fff.lua @@ -0,0 +1 @@ +return require('fff.main') diff --git a/lua/fff/conf.lua b/lua/fff/conf.lua new file mode 100644 index 0000000..ab0683f --- /dev/null +++ b/lua/fff/conf.lua @@ -0,0 +1,493 @@ +local M = {} + +--- @class FffLayoutConfig +--- @field height number +--- @field width number +--- @field prompt_position string +--- @field preview_position string +--- @field preview_size number +--- @field min_list_height number +--- @field show_scrollbar boolean +--- @field path_shorten_strategy string +--- @field border? 'single'|'double'|'rounded'|'solid'|'shadow'|'none' Border preset; falls back to `vim.o.winborder` when nil + +--- @class FffPreviewConfig +--- @field enabled boolean +--- @field max_size number +--- @field chunk_size number +--- @field binary_file_threshold number +--- @field imagemagick_info_format_str string +--- @field line_numbers boolean +--- @field cursorlineopt string +--- @field wrap_lines boolean +--- @field filetypes table + +--- @class FffKeymapsConfig +--- @field close string +--- @field select string +--- @field select_split string +--- @field select_vsplit string +--- @field select_tab string +--- @field move_up string|string[] +--- @field move_down string|string[] +--- @field preview_scroll_up string +--- @field preview_scroll_down string +--- @field toggle_debug string +--- @field cycle_grep_modes string +--- @field cycle_previous_query string +--- @field cycle_forward_query string +--- @field grep_jump_to_next_file string|string[] +--- @field grep_jump_to_prev_file string|string[] +--- @field toggle_select string +--- @field send_to_quickfix string +--- @field focus_list string +--- @field focus_preview string + +--- @class FffFrecencyConfig +--- @field enabled boolean +--- @field db_path string + +--- @class FffHistoryConfig +--- @field enabled boolean +--- @field db_path string +--- @field min_combo_count number +--- @field combo_boost_score_multiplier number + +--- @class FffGrepConfig +--- @field max_file_size number +--- @field max_matches_per_file number +--- @field smart_case boolean +--- @field time_budget_ms number +--- @field modes string[] +--- @field trim_whitespace boolean +--- @field location_format string + +--- @alias FffSelectAction 'edit' | 'split' | 'vsplit' | 'tab' + +--- @class FffSelectConfig +--- @field select_window fun(current_buf: integer, action: FffSelectAction): integer|nil + +--- @class FffConfig +--- @field base_path string +--- @field prompt string +--- @field title string +--- @field max_results number +--- @field max_threads number +--- @field lazy_sync boolean +--- @field prompt_vim_mode boolean +--- @field follow_symlinks boolean +--- @field enable_fs_root_scanning boolean +--- @field enable_home_dir_scanning boolean +--- @field layout FffLayoutConfig +--- @field preview FffPreviewConfig +--- @field keymaps FffKeymapsConfig +--- @field hl table +--- @field frecency FffFrecencyConfig +--- @field history FffHistoryConfig +--- @field select FffSelectConfig +--- @field git table +--- @field debug table +--- @field logging table +--- @field wrap_around boolean +--- @field file_picker table +--- @field grep FffGrepConfig + +---@class fff.conf.State +local state = { + ---@type FffConfig|nil + config = nil, +} + +local DEPRECATION_RULES = { + { + -- Top-level width -> layout.width + old_path = { 'width' }, + new_path = { 'layout', 'width' }, + message = 'config.width is deprecated. Use config.layout.width instead.', + }, + { + -- Top-level height -> layout.height + old_path = { 'height' }, + new_path = { 'layout', 'height' }, + message = 'config.height is deprecated. Use config.layout.height instead.', + }, + { + -- preview.width -> layout.preview_size + old_path = { 'preview', 'width' }, + new_path = { 'layout', 'preview_size' }, + message = 'config.preview.width is deprecated. Use config.layout.preview_size instead.', + }, + { + -- layout.preview_width -> layout.preview_size + old_path = { 'layout', 'preview_width' }, + new_path = { 'layout', 'preview_size' }, + message = 'config.layout.preview_width is deprecated. Use config.layout.preview_size instead.', + }, +} + +--- Get value from nested table using path array +--- @param tbl table Source table +--- @param path table Array of keys to traverse +--- @return any|nil Value at path or nil if not found +local function get_nested_value(tbl, path) + local current = tbl + for _, key in ipairs(path) do + if type(current) ~= 'table' or current[key] == nil then return nil end + current = current[key] + end + + return current +end + +--- Set value in nested table using path array, creating intermediate tables +--- @param tbl table Target table +--- @param path table Array of keys to traverse +--- @param value any Value to set +local function set_nested_value(tbl, path, value) + local current = tbl + for i = 1, #path - 1 do + local key = path[i] + if type(current[key]) ~= 'table' then current[key] = {} end + current = current[key] + end + + current[path[#path]] = value +end + +--- Remove value from nested table using path array +--- @param tbl table Target table +--- @param path table Array of keys to traverse +local function remove_nested_value(tbl, path) + if #path == 0 then return end + + local current = tbl + for i = 1, #path - 1 do + local key = path[i] + if type(current[key]) ~= 'table' then return end + current = current[key] + end + + current[path[#path]] = nil +end + +--- Handle deprecated configuration options with migration warnings +--- @param user_config table User provided configuration +--- @return table Migrated configuration +local function handle_deprecated_config(user_config) + if not user_config then return {} end + + local migrated_config = vim.deepcopy(user_config) + + for _, rule in ipairs(DEPRECATION_RULES) do + local old_value = get_nested_value(user_config, rule.old_path) + if old_value ~= nil then + set_nested_value(migrated_config, rule.new_path, old_value) + remove_nested_value(migrated_config, rule.old_path) + + vim.notify('FFF: ' .. rule.message, vim.log.levels.WARN) + end + end + + return migrated_config +end + +---@param name table list of highlight groups to choose from +---@return string one of the provided groups +local function fallback_hl(name) + local resolved_hl + for _, hl in ipairs(name) do + local resolved_group = vim.api.nvim_get_hl(0, { name = hl }) + + if not vim.tbl_isempty(resolved_group) then resolved_hl = hl end + end + + return resolved_hl or name[#name] +end + +local function init() + local config = vim.g.fff or {} + local default_config = { + base_path = vim.fn.getcwd(), + prompt = '🪿 ', + title = 'FFFiles', + max_results = 100, + max_threads = 4, + lazy_sync = true, -- set to false if you want file indexing to start on open + prompt_vim_mode = false, -- set to true to enable vim-mode in the prompt: leaves insert for normal mode bindings (also allows p or l to jump around) the second closes the picker + wrap_around = false, -- set to true to wrap cursor to the opposite end when reaching the first/last item + follow_symlinks = false, -- set to true to follow symbolic links during file indexing + -- Allow fff in the user's $HOME director. + enable_home_dir_scanning = true, + -- Allow fff in a filesystem root (e.g. `/`, `C:\`) + enable_fs_root_scanning = false, + layout = { + height = 0.8, + width = 0.8, + prompt_position = 'bottom', -- or 'top' + preview_position = 'right', -- or 'left', 'right', 'top', 'bottom' + preview_size = 0.5, + -- Border style for the picker windows: 'single', 'double', 'rounded', + -- 'solid', 'shadow' or 'none'. Leave unset (nil) to follow the global + -- `vim.o.winborder` setting. + border = nil, + flex = { -- set to nil to disable flex layout + size = 130, -- column threshold: if screen width >= size, use preview_position; otherwise use wrap + wrap = 'top', -- position to use when screen is narrower than size + }, + -- Minimum list height required to render the preview. When the available + -- list area would drop below this on small terminals, the preview is + -- auto-hidden so the file list stays usable. Set to 0 to disable. + min_list_height = 10, + show_scrollbar = true, -- Show scrollbar for pagination + -- How to shorten long directory paths in the file list: + -- 'middle' (default): always uses dots (a/./b, a/../b, a/.../b) + -- 'middle_number' uses dots for 1-3 hidden (a/./b, a/../b, a/.../b) + -- and numbers for 4+ (a/.4./b, a/.5./b) + -- 'end': truncates from the end, keeps the start (home/user/projects) + -- 'start': truncates from the start, keeps the end (.../parts/ai_extracted) + path_shorten_strategy = 'middle', + }, + preview = { + enabled = true, + max_size = 10 * 1024 * 1024, -- Do not try to read files larger than 10MB + chunk_size = 8192, -- Bytes per chunk for dynamic loading (8kb - fits ~100-200 lines) + binary_file_threshold = 1024, -- amount of bytes to scan for binary content (set 0 to disable) + imagemagick_info_format_str = '%m: %wx%h, %[colorspace], %q-bit', + line_numbers = false, + cursorlineopt = 'both', + wrap_lines = false, + filetypes = { + svg = { wrap_lines = true }, + markdown = { wrap_lines = true }, + text = { wrap_lines = true }, + }, + }, + keymaps = { + close = '', + select = '', + select_split = '', + select_vsplit = '', + select_tab = '', + -- you can assign multiple keys to any action + move_up = { '', '' }, + move_down = { '', '' }, + preview_scroll_up = '', + preview_scroll_down = '', + toggle_debug = '', + -- grep mode: cycle between plain text, regex, and fuzzy search + cycle_grep_modes = '', + -- grep mode only: jump cursor to first item of next/prev file group + grep_jump_to_next_file = { '', '' }, + grep_jump_to_prev_file = { '', '' }, + -- goes to the previous query in history + cycle_previous_query = '', + -- goes to the next query in history (forward) + cycle_forward_query = '', + -- multi-select keymaps for quickfix + toggle_select = '', + send_to_quickfix = '', + -- this are specific for the normal mode (you can exit it using any other keybind like jj) + focus_list = 'l', + focus_preview = 'p', + }, + hl = { + border = 'FloatBorder', + normal = 'NormalFloat', + matched = 'IncSearch', + title = 'Title', + prompt = 'Question', + cursor = fallback_hl({ 'CursorLine', 'Visual' }), + frecency = 'Number', + debug = 'Comment', + combo_header = 'Number', + scrollbar = 'Comment', + directory_path = 'Comment', + -- Multi-select highlights + selected = 'FFFSelected', + selected_active = 'FFFSelectedActive', + -- Git text highlights for file names + git_staged = 'FFFGitStaged', + git_modified = 'FFFGitModified', + git_deleted = 'FFFGitDeleted', + git_renamed = 'FFFGitRenamed', + git_untracked = 'FFFGitUntracked', + git_ignored = 'FFFGitIgnored', + -- Git sign/border highlights + git_sign_staged = 'FFFGitSignStaged', + git_sign_modified = 'FFFGitSignModified', + git_sign_deleted = 'FFFGitSignDeleted', + git_sign_renamed = 'FFFGitSignRenamed', + git_sign_untracked = 'FFFGitSignUntracked', + git_sign_ignored = 'FFFGitSignIgnored', + -- Git sign selected highlights + git_sign_staged_selected = 'FFFGitSignStagedSelected', + git_sign_modified_selected = 'FFFGitSignModifiedSelected', + git_sign_deleted_selected = 'FFFGitSignDeletedSelected', + git_sign_renamed_selected = 'FFFGitSignRenamedSelected', + git_sign_untracked_selected = 'FFFGitSignUntrackedSelected', + git_sign_ignored_selected = 'FFFGitSignIgnoredSelected', + -- Grep highlights + grep_match = 'IncSearch', -- Highlight for matched text in grep results + grep_line_number = 'LineNr', -- Highlight for :line:col location + grep_regex_active = 'DiagnosticInfo', -- Highlight for keybind + label when regex is on + grep_plain_active = 'Comment', -- Highlight for keybind + label when regex is off + grep_fuzzy_active = 'DiagnosticHint', -- Highlight for keybind + label when fuzzy is on + -- Cross-mode suggestion highlights + suggestion_header = 'WarningMsg', -- Highlight for the "No results found. Suggested..." banner + -- File info panel highlights + file_info_section = 'FFFFileInfoSection', -- Section header label (e.g. "file", "score") + file_info_separator = 'FFFFileInfoSeparator', -- Dash dividers used like a border + file_info_label = 'FFFFileInfoLabel', -- Row labels (Size, Type, Git, ...) + file_info_value = 'FFFFileInfoValue', -- Plain values + file_info_value_dim = 'FFFFileInfoValueDim', -- Tertiary values, separators inside rows + file_info_size = 'FFFFileInfoSize', -- File size value + file_info_type = 'FFFFileInfoType', -- Filetype value + file_info_path = 'FFFFileInfoPath', -- Full path value + file_info_total_score = 'FFFFileInfoTotalScore', -- Total score (bold) + file_info_match_type = 'FFFFileInfoMatchType', -- match_type label (bold) + file_info_score_pos = 'FFFFileInfoScorePos', -- Positive score components + file_info_score_neg = 'FFFFileInfoScoreNeg', -- Negative score components / penalties + -- Per-window 'winhighlight' overrides. When nil, falls back to a combination of `normal`, `border`, and `title` above. + -- Accepts either a string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, `file_info` keys. + -- Example: `winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title'` + -- Example: `winhl = { prompt = 'Normal:Pmenu,...', list = 'Normal:NormalFloat,...' }` + winhl = nil, + }, + -- Store file open frecency + frecency = { + enabled = true, + db_path = vim.fn.stdpath('cache') .. '/fff_nvim', + }, + -- Store successfully opened queries with respective matches + history = { + enabled = true, + db_path = vim.fn.stdpath('data') .. '/fff_queries', + min_combo_count = 3, -- Minimum selections before combo boost applies (3 = boost starts on 3rd selection) + combo_boost_score_multiplier = 100, -- Score multiplier for combo matches (files repeatedly opened with same query) + }, + select = { + --- Returns winid to open the file in. Return nil to open in the invoking + --- window. Default retargets when the invoking window can't host a file + --- buffer (special buftype, non-modifiable, or winfixbuf). + --- @param current_buf integer + --- @param action FffSelectAction + --- @return integer|nil + select_window = function(current_buf, action) + if action ~= 'edit' then return nil end + local current_win = vim.api.nvim_get_current_win() + local buftype = vim.api.nvim_get_option_value('buftype', { buf = current_buf }) + local modifiable = vim.api.nvim_get_option_value('modifiable', { buf = current_buf }) + local winfixbuf = require('fff.utils').window_has_winfixbuf(current_win) + if buftype == '' and modifiable and not winfixbuf then return nil end + return require('fff.utils').find_suitable_window() + end, + }, + -- Git integration + git = { + status_text_color = false, -- Apply git status colors to filename text (default: false, only sign column) + }, + debug = { + enabled = false, -- Show file info panel in preview + show_scores = false, -- Show scores inline in the UI + show_file_info = { + file_info = true, -- Size, type, git status, frecency + score_breakdown = true, -- Total + match type, bonuses, modifiers, penalty + -- Modified + accessed timestamps. Pass a boolean to toggle the + -- whole section, or a table to hide individual rows: + -- timings = { modified = false, accessed = true } + timings = true, + full_path = true, -- Full absolute path at the bottom + }, + }, + logging = { + enabled = true, + -- Path-shape hint: each nvim startup writes a fresh sibling file + -- `++.` next to this path. The literal + -- path itself is never written to — multiple concurrent nvim instances + -- get their own per-pid file with no locking. + log_file = vim.fn.stdpath('log') .. '/fff.log', + log_level = 'info', + -- How many session log files to retain. Newest are kept, older are + -- pruned on the next startup. Set to 0 to disable retention. + retain_runs = 20, + }, + -- find_files settings + file_picker = { + current_file_label = '(current)', + }, + -- grep settings + grep = { + max_file_size = 10 * 1024 * 1024, -- Skip files larger than 10MB + max_matches_per_file = 100, -- Maximum matches per file (set 0 to unlimited) + smart_case = true, -- Case-insensitive unless query has uppercase + time_budget_ms = 150, -- Max search time in ms per call (prevents UI freeze, 0 = no limit) + modes = { 'plain', 'regex', 'fuzzy' }, -- Available grep modes and their cycling order + trim_whitespace = false, -- Strip leading whitespace from matched lines (useful for cleaner display) + -- Treat filename-like tokens (e.g. `score.rs`, `src/main.rs`) in a grep query as a + -- file-path filter, scoping the content search to matching files. When off, such + -- tokens are searched as literal text. A token is a filename if it has a valid-looking + -- extension and no wildcards. + enable_filename_constraint = false, + -- Format string for the line/column location prefix in grep results. + -- Uses vim's printf-style format: %d placeholders for line and column (1-based). + -- Default ':%d:%d' renders as ':356:1'. Use ':%d' for line-only ':356'. + location_format = ':%d:%d', + }, + } + + local migrated_user_config = handle_deprecated_config(config) + local merged_config = vim.tbl_deep_extend('force', default_config, migrated_user_config) + + -- Normalise show_file_info: accept a boolean shorthand or a partial table + local sfi = merged_config.debug and merged_config.debug.show_file_info + local default_sections = { file_info = true, score_breakdown = true, timings = true, full_path = true } + if type(sfi) == 'boolean' then + merged_config.debug.show_file_info = { + file_info = sfi, + score_breakdown = sfi, + timings = sfi, + full_path = sfi, + } + elseif type(sfi) == 'table' then + for k, v in pairs(default_sections) do + if sfi[k] == nil then sfi[k] = v end + end + else + merged_config.debug.show_file_info = default_sections + end + + state.config = merged_config +end + +--- Setup the file picker with the given configuration +--- @param config FffConfig Configuration options +function M.setup(config) vim.g.fff = config end + +--- @return FffConfig the fff configuration +function M.get() + if not state.config then init() end + return state.config +end + +--- True when preview rendering is requested by config. Defaults to `true` +--- when `config` (or its `preview` block) is missing so callers don't have +--- to guard against partial state during init. +--- @param config? FffConfig Optional config; falls back to `M.get()` when nil. +--- @return boolean +function M.preview_enabled(config) + config = config or M.get() + if not config or not config.preview then return true end + return config.preview.enabled +end + +--- @return boolean state_changed +function M.toggle_debug() + local old_debug_state = state.config.debug.show_scores + state.config.debug.show_scores = not state.config.debug.show_scores + state.config.debug.enabled = state.config.debug.show_scores + local status = state.config.debug.show_scores and 'enabled' or 'disabled' + vim.notify('FFF debug scores ' .. status, vim.log.levels.INFO) + return old_debug_state ~= state.config.debug.show_scores +end + +return M diff --git a/lua/fff/core.lua b/lua/fff/core.lua new file mode 100644 index 0000000..1b3f3a1 --- /dev/null +++ b/lua/fff/core.lua @@ -0,0 +1,176 @@ +local fuzzy = require('fff.fuzzy') +if not fuzzy then error('Failed to load fff.fuzzy module. Ensure the Rust backend is compiled and available.') end + +local M = {} + +---@class fff.core.State +local state = { + ---@type boolean + initialized = false, + ---@type boolean + file_picker_initialized = false, +} + +---@param config table +local function setup_global_autocmds(config) + local group = vim.api.nvim_create_augroup('fff_file_tracking', { clear = true }) + + if config.frecency.enabled then + vim.api.nvim_create_autocmd({ 'BufEnter' }, { + group = group, + desc = 'Track file access for FFF frecency', + callback = function(args) + local file_path = args.file + if not (file_path and file_path ~= '' and not vim.startswith(file_path, 'term://')) then return end + + vim.uv.fs_stat(file_path, function(err, stat) + if err or not stat then return end + + vim.uv.fs_realpath(file_path, function(rp_err, real_path) + if rp_err or not real_path then return end + local ok, track_err = pcall(fuzzy.track_access, real_path) + + if not ok then + vim.schedule( + function() vim.notify('FFF: Failed to track file access: ' .. tostring(track_err), vim.log.levels.ERROR) end + ) + end + end) + end) + end, + }) + end + + -- make sure that this won't work correctly if autochdir plugins are enabled + -- using a pure :cd command but will work using lua api or :e command + vim.api.nvim_create_autocmd('DirChanged', { + group = group, + callback = function() + -- Window-local `:lcd` / `:tcd` are per-window — they don't change the + -- effective project root for the picker, so bail before touching + -- anything else. + if vim.v.event.scope == 'window' then return end + if not state.initialized then return end + + local new_cwd = vim.v.event.cwd + if not new_cwd or new_cwd == '' then return end + + -- Canonicalize both sides before comparing. `vim.v.event.cwd` is + -- whatever the caller passed to `:cd` (often unexpanded, sometimes + -- containing `~` or symlinks), while `config.base_path` is the form + -- the picker was last re-indexed against (post-`expand`). Without + -- resolving symlinks + ensuring an absolute path, trivially + -- equivalent paths compare as different (`/private/var/x` vs + -- `/var/x` on macOS, resolved-vs-unresolved symlinks from LSP root + -- detection, etc.) and every such mismatch schedules a 450k-file + -- reindex through the Rust side. + local function canonicalize(p) + if not p or p == '' then return p end + local abs = vim.fn.fnamemodify(vim.fn.expand(p), ':p') + -- `:p` leaves a trailing slash on directories — strip for + -- comparison stability. + abs = abs:gsub('/+$', '') + local ok, resolved = pcall(vim.fn.resolve, abs) + return (ok and resolved ~= '') and resolved or abs + end + + local new_canonical = canonicalize(new_cwd) + local base_canonical = canonicalize(config.base_path) + if new_canonical == base_canonical then return end + + vim.schedule(function() + local change_ok, err = pcall(M.change_indexing_directory, new_canonical) + if not change_ok then + vim.notify('FFF: Failed to change indexing directory: ' .. tostring(err), vim.log.levels.ERROR) + end + end) + end, + desc = 'Automatically sync FFF directory changes', + }) +end + +--- @return boolean +M.is_file_picker_initialized = function() return state.file_picker_initialized end + +--- Change the base directory for the file picker. Triggers a reindex on the +--- Rust side and updates `config.base_path` so subsequent `:cd` events compare +--- against the new root. +--- @param new_path string New directory path to use as base +--- @return boolean ok `true` if the reindex was scheduled, `false` otherwise +M.change_indexing_directory = function(new_path) + if not new_path or new_path == '' then + vim.notify('Directory path is required', vim.log.levels.ERROR) + return false + end + + local expanded_path = vim.fn.expand(new_path) + if vim.fn.isdirectory(expanded_path) ~= 1 then + vim.notify('Directory does not exist: ' .. expanded_path, vim.log.levels.ERROR) + return false + end + + local fff_rust = M.ensure_initialized() + local config = require('fff.conf').get() + local ok, err = pcall(fff_rust.restart_index_in_path, expanded_path, { + follow_symlinks = config.follow_symlinks, + enable_fs_root_scanning = config.enable_fs_root_scanning, + enable_home_dir_scanning = config.enable_home_dir_scanning, + enable_filename_constraint = config.grep and config.grep.enable_filename_constraint, + }) + if not ok then + vim.notify('Failed to change directory: ' .. err, vim.log.levels.ERROR) + return false + end + + require('fff.conf').get().base_path = expanded_path + return true +end + +M.ensure_initialized = function() + if state.initialized then return fuzzy end + state.initialized = true + + local config = require('fff.conf').get() + if config.logging.enabled then + local log_success, log_error = + pcall(fuzzy.init_tracing, config.logging.log_file, config.logging.log_level, config.logging.retain_runs) + if log_success then + M.log_file_path = log_error + else + vim.notify('Failed to initialize logging: ' .. (tostring(log_error) or 'unknown error'), vim.log.levels.WARN) + end + end + + local frecency_db_path = config.frecency.db_path or (vim.fn.stdpath('cache') .. '/fff_frecency') + local history_db_path = config.history.db_path or (vim.fn.stdpath('data') .. '/fff_history') + + local ok, result = pcall(fuzzy.init_db, frecency_db_path, history_db_path, true) + if not ok then vim.notify('Failed to databases: ' .. tostring(result), vim.log.levels.WARN) end + + ok, result = pcall(fuzzy.init_file_picker, config.base_path, { + follow_symlinks = config.follow_symlinks, + enable_fs_root_scanning = config.enable_fs_root_scanning, + enable_home_dir_scanning = config.enable_home_dir_scanning, + enable_filename_constraint = config.grep and config.grep.enable_filename_constraint, + }) + if not ok then + vim.notify('Failed to initialize file picker: ' .. tostring(result), vim.log.levels.ERROR) + return fuzzy + end + + state.file_picker_initialized = true + setup_global_autocmds(config) + + local highlights = require('fff.highlights') + highlights.setup() + + vim.api.nvim_create_autocmd('ColorScheme', { + group = vim.api.nvim_create_augroup('fff_highlights', { clear = true }), + callback = function() highlights.setup() end, + desc = 'Re-apply FFF highlights on colorscheme change', + }) + + return fuzzy +end + +return M diff --git a/lua/fff/download.lua b/lua/fff/download.lua new file mode 100644 index 0000000..175dc4c --- /dev/null +++ b/lua/fff/download.lua @@ -0,0 +1,289 @@ +local M = {} +local system = require('fff.utils.system') +local fs_utils = require('fff.utils.fs') +local fff_version = require('fff.utils.version') + +local GITHUB_REPO = 'dmtrKovalenko/fff.nvim' + +local function get_binary_dir(plugin_dir) return plugin_dir .. '/../target/release' end + +local function get_binary_path(plugin_dir) + local binary_dir = get_binary_dir(plugin_dir) + local extension = system.get_lib_extension() + return binary_dir .. '/libfff_nvim.' .. extension +end + +local function binary_exists(plugin_dir) + local binary_path = get_binary_path(plugin_dir) + local stat = vim.uv.fs_stat(binary_path) + if stat and stat.type == 'file' then return true end + + -- On Windows the rename over a loaded DLL fails, so a verified binary may be + -- left at binary_path .. '.tmp'. Promote it now that the old session is gone. + local tmp_path = binary_path .. '.tmp' + local tmp_stat = vim.uv.fs_stat(tmp_path) + if tmp_stat and tmp_stat.type == 'file' then + -- Verify the .tmp is a valid library before promoting it, in case the + -- process was killed between the loadlib check and the rename attempt + -- during a previous download, leaving a corrupt or partial .tmp on disk. + local loader = package.loadlib(tmp_path, 'luaopen_fff_nvim') + if not loader then + vim.uv.fs_unlink(tmp_path) + return false + end + local ok = vim.uv.fs_rename(tmp_path, binary_path) + return ok ~= nil + end + + return false +end + +local function download_file(url, output_path, opts, callback) + opts = opts or {} + + local dir = vim.fn.fnamemodify(output_path, ':h') + fs_utils.mkdir_recursive(dir, function(mkdir_ok, mkdir_err) + if not mkdir_ok then + callback(false, mkdir_err) + return + end + + local curl_args = { + 'curl', + '--fail', + '--location', + '--silent', + '--show-error', + '--output', + output_path, + } + + if opts.proxy then + table.insert(curl_args, '--proxy') + table.insert(curl_args, opts.proxy) + end + + if opts.extra_curl_args then + for _, arg in ipairs(opts.extra_curl_args) do + table.insert(curl_args, arg) + end + end + + table.insert(curl_args, url) + vim.system(curl_args, {}, function(result) + if result.code ~= 0 then + callback(false, 'Failed to download: ' .. (result.stderr or 'unknown error')) + return + end + callback(true, nil) + end) + end) +end + +local function download_from_github(version, binary_path, opts, callback) + opts = opts or {} + + local triple = system.get_triple() + local extension = system.get_lib_extension() + local binary_name = triple .. '.' .. extension + local url = string.format('https://github.com/%s/releases/download/%s/%s', GITHUB_REPO, version, binary_name) + + vim.schedule(function() + vim.notify(string.format('Downloading fff.nvim binary for ' .. version), vim.log.levels.INFO) + vim.notify(string.format('Do not open fff until you see a success notification.'), vim.log.levels.WARN) + end) + + -- Download to a temp path first so we can validate before replacing the live binary. + -- If we wrote directly to binary_path and the current process already has the old + -- library loaded, package.loadlib() on the same path returns the *cached* handle — + -- meaning a truncated download would pass validation silently. + -- Using a distinct temp path forces dlopen to load the new file for real. + local tmp_path = binary_path .. '.tmp' + + download_file(url, tmp_path, { + proxy = opts.proxy, + extra_curl_args = opts.extra_curl_args, + }, function(success, err) + if not success then + vim.uv.fs_unlink(tmp_path) + callback(false, err) + return + end + + vim.schedule(function() + -- Validate the downloaded binary by actually loading it (temp path is not yet + -- loaded by this process, so dlopen loads the new file for real and catches + -- truncated or corrupt downloads). + -- Note: package.loadlib returns (nil, error_string) on failure rather than throwing. + local loader, load_err = package.loadlib(tmp_path, 'luaopen_fff_nvim') + + if not loader then + vim.uv.fs_unlink(tmp_path) + callback(false, 'Downloaded binary is not valid: ' .. (load_err or 'unknown error')) + return + end + + -- Atomically replace the live binary only after successful validation. + -- On Windows the old .dll may be locked by the current process, so rename can + -- fail if fff is already loaded. In that case, leave the verified .tmp on disk + -- so the next Neovim start can pick it up automatically. + local rename_ok, rename_err = vim.uv.fs_rename(tmp_path, binary_path) + if not rename_ok then + if vim.uv.os_uname().sysname:lower():match('windows') then + vim.notify( + 'fff.nvim binary downloaded to ' + .. tmp_path + .. '.\nThe live binary is locked by the current session — please restart Neovim to apply the update.', + vim.log.levels.WARN + ) + callback(true, nil) + else + vim.uv.fs_unlink(tmp_path) + callback(false, 'Failed to install binary: ' .. (rename_err or 'unknown error')) + end + return + end + + vim.notify('fff.nvim binary downloaded successfully!', vim.log.levels.INFO) + callback(true, nil) + end) + end) +end + +function M.ensure_downloaded(opts, callback) + opts = opts or {} + local plugin_dir = vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':h:h') + + if binary_exists(plugin_dir) and not opts.force then + callback(true, nil) + return + end + + local function on_release_tag(release_tag) + if not release_tag then + callback(false, 'Could not determine target version') + return + end + + local binary_path = get_binary_path(plugin_dir) + download_from_github(release_tag, binary_path, opts, callback) + end + + if opts.version then + on_release_tag(opts.version) + else + -- plugin_dir is /lua; parent is the repo root + local repo_root = vim.fn.fnamemodify(plugin_dir, ':h') + + -- 1. Try reading the CI-created tag on HEAD (no version computation) + local tag = fff_version.current_release_tag(repo_root) + if tag then + on_release_tag(tag) + return + end + + -- 2. No local tag — construct the nightly version (bumps patch so + -- the prerelease is higher than Cargo.toml base in semver) + local info, err = fff_version.resolve(repo_root) + if info then + on_release_tag(info.release_tag) + return + end + + callback(false, err or 'Could not determine target version') + end +end + +function M.download_binary(callback) + M.ensure_downloaded({ force = true }, function(success, err) + if not success then + if callback then + callback(false, err) + else + vim.schedule( + function() + vim.notify('Failed to download fff.nvim binary: ' .. (err or 'unknown error'), vim.log.levels.ERROR) + end + ) + end + return + end + if callback then callback(true, nil) end + end) +end + +function M.build_binary(callback) + local plugin_dir = vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':h:h') + local has_rustup = vim.fn.executable('rustup') == 1 + if not has_rustup then + callback( + false, + 'rustup is not found. It is required to build the fff.nvim binary. Install it from https://rustup.rs/' + ) + return + end + + vim.system({ 'cargo', 'build', '--release' }, { cwd = plugin_dir }, function(result) + if result.code ~= 0 then + callback(false, 'Failed to build rust binary: ' .. (result.stderr or 'unknown error')) + return + end + callback(true, nil) + end) +end + +function M.download_or_build_binary() + local done = false + local fatal_error = nil + + M.ensure_downloaded({ force = true }, function(download_success, download_error) + if download_success then + done = true + return + end + + vim.schedule( + function() + vim.notify( + 'Error downloading binary: ' .. (download_error or 'unknown error') .. '\nTrying cargo build --release\n', + vim.log.levels.WARN + ) + end + ) + + M.build_binary(function(build_success, build_error) + if not build_success then + fatal_error = 'Failed to build fff.nvim binary. Build error: ' .. (build_error or 'unknown error') + else + vim.schedule(function() vim.notify('fff.nvim binary built successfully!', vim.log.levels.INFO) end) + end + done = true + end) + end) + + -- Block the caller (and keep the Neovim event loop alive) until the entire + -- download-or-build chain finishes. This is critical for lazy.nvim build + -- hooks: lazy returns from the hook immediately after this function returns, + -- and if Neovim exits before the final rename(tmp → libfff_nvim.{dylib,so,dll}) + -- executes, the binary is never written to disk. vim.wait pumps the event + -- loop so all vim.system / vim.schedule callbacks can fire. + local timeout_ms = 1000 * 60 * 2 -- 2 minutes + local ok, wait_err = vim.wait(timeout_ms, function() return done end, 100) + if not ok and wait_err == -2 then error('fff.nvim: download_or_build_binary timed out') end + + if fatal_error then error(fatal_error) end +end + +function M.get_binary_path() + local plugin_dir = vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':h:h') + return get_binary_path(plugin_dir) +end + +function M.get_binary_cpath_component() + local plugin_dir = vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':h:h') + local binary_dir = get_binary_dir(plugin_dir) + local extension = system.get_lib_extension() + return binary_dir .. '/lib?.' .. extension +end + +return M diff --git a/lua/fff/file_picker/file_info.lua b/lua/fff/file_picker/file_info.lua new file mode 100644 index 0000000..52eace9 --- /dev/null +++ b/lua/fff/file_picker/file_info.lua @@ -0,0 +1,415 @@ +local M = {} + +---@class FFFFileInfoExtmark +---@field row integer +---@field col integer +---@field end_col integer|nil +---@field hl_group string|nil + +---@class FFFFileInfoResult +---@field lines string[] +---@field extmarks FFFFileInfoExtmark[] +---@field height integer + +---@class FFFFileInfoSections +---@field file_info boolean +---@field score_breakdown boolean +---@field timings boolean +---@field full_path boolean + +---@class FFFFileInfoFile +---@field relative_path string +---@field absolute_path string|nil +---@field size_formatted string +---@field filetype string +---@field git_status string +---@field access_frecency_score integer +---@field modification_frecency_score integer +---@field times_opened integer +---@field modified_formatted string +---@field accessed_formatted string + +---@class FFFFileInfoScore +---@field total integer +---@field match_type string +---@field base_score integer +---@field filename_bonus integer +---@field special_filename_bonus integer +---@field frecency_boost integer +---@field combo_match_boost integer +---@field distance_penalty integer +---@field current_file_penalty integer + +local Builder = {} +Builder.__index = Builder + +function Builder.new() return setmetatable({ lines = {}, extmarks = {} }, Builder) end + +function Builder:add_line(text) table.insert(self.lines, text or '') end + +function Builder:add_hl(row, col, end_col, hl_group) + if not hl_group or hl_group == '' then return end + table.insert(self.extmarks, { row = row, col = col, end_col = end_col, hl_group = hl_group }) +end + +-- Section header: `─