commit e90a96653637a994d3cb9cf4fb71415039d301e6 Author: wehub-resource-sync Date: Mon Jul 13 12:29:01 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..998ede1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [Andyyyy64] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..dd15b5e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,46 @@ +name: Bug Report +description: Report a bug or unexpected behavior +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug! Please fill out the information below. + - type: textarea + id: description + attributes: + label: Description + description: What happened? What did you expect to happen? + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Steps to Reproduce + description: How can we reproduce this issue? + placeholder: | + 1. Run `whichllm --gpu "RTX 4090"` + 2. ... + validations: + required: true + - type: textarea + id: hardware + attributes: + label: Hardware Info + description: Paste the output of `whichllm hardware` + render: shell + - type: input + id: python-version + attributes: + label: Python Version + placeholder: "3.11" + - type: input + id: os + attributes: + label: Operating System + placeholder: "Ubuntu 24.04 / macOS 15 / Windows 11" + - type: input + id: version + attributes: + label: whichllm Version + placeholder: "0.3.0" diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..6b2057f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,27 @@ +name: Feature Request +description: Suggest a new feature or improvement +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for suggesting a feature! Please describe what you'd like to see. + - type: textarea + id: problem + attributes: + label: Problem + description: What problem does this feature solve? + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed Solution + description: How would you like this to work? + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: Any alternative solutions you've considered? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..a29231d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## What + + + +## Why + + + +## Testing + +- [ ] Tests pass (`pytest`) +- [ ] New tests added (if applicable) +- [ ] Tested on real hardware (if hardware-related) + +## Notes + + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..e564153 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,25 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff linter + run: ruff check . + - name: Run Ruff formatter check + run: ruff format --check . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..057ba70 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,27 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +permissions: + id-token: write + +jobs: + publish: + runs-on: ubuntu-latest + environment: pypi + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install build tools + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e1f2fec --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install pytest pyarrow + - name: Run tests + run: pytest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b51978 --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg +*.egg-info/ +dist/ +build/ +*.whl + +# Virtual environments +.venv/ +venv/ +env/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Environment +.env +.env.local +.env.*.local + +# Logs +*.log +.claude/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2e91f53 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..53423cf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,429 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## [0.5.15] - 2026-07-03 + +### Changed + +- Split the oversized Hugging Face model fetcher into focused modules while + keeping the existing `whichllm.models.fetcher` import surface. (#41, #140) + +### Fixed + +- Output now resolves ranked GGUF recommendations to the actual downloadable + artifact repo and filename when the ranked base model and runnable GGUF live + in different Hugging Face repos. (#137, #138) +- AA index scoring now uses retuned normalization bounds and a refreshed + fallback snapshot for the reworked Artificial Analysis scale. (#101, #139) +- Invalid ranking filters such as `--top 0`, negative `--min-speed`, and + negative `--min-params` now fail with clear errors instead of silently + producing wrong output. (#142) + +## [0.5.14] - 2026-06-29 + +### Added + +- Added sliding-window attention metadata to model fetching and KV cache + estimation, improving VRAM estimates for models that use SWA. (#124) +- Added curated Intel Arc Pro B70 / Battlemage G31 detection and simulation, + including the `0xe223` PCI device ID and 32 GB VRAM / 608 GB/s bandwidth + defaults. (#93, #136) + +### Fixed + +- Model and benchmark metadata fetches now request `gzip, deflate` instead of + brotli, avoiding broken `br` responses from mirrors or intermediate servers. + (#128, #136) + +## [0.5.13] - 2026-06-25 + +### Added + +- Added `HF_ENDPOINT` support for Hugging Face model metadata fetches, so users + behind a mirror can point whichllm at a compatible Hub endpoint. (#128, #131) +- Added manual detected-GPU overrides for usable VRAM and bandwidth, which helps + iGPU and unified-memory systems where automatic detection is too conservative. + (#132, #133) +- README now points users toward safer first-run flags when they want + full-GPU, usable-speed recommendations with extra VRAM headroom. + +### Fixed + +- Search terms such as `7B`, `0.5B`, and `500M` now match model parameter size + instead of plain substrings, so `qwen 7b` no longer returns `1.7B` or + `30B-A3B` models by accident. (#107, #126) +- GGUF sizing now treats FP16 and ternary `TQ1_0` / `TQ2_0` quant types + correctly, avoiding underestimates for those files. (#125) + +## [0.5.12] - 2026-06-18 + +### Added + +- Default ranking tables now show memory required, estimated generation speed, + fit type, and published date. Download counts are still available with + `--details`. +- Added `--speed any|usable|fast` as named generation-speed filters while + keeping `--min-speed` for exact tok/s thresholds. +- Added `--fit gpu` as a natural alias for full-GPU-only recommendations. +- Added `--markdown` / `-m` for pasteable GitHub-Flavored Markdown ranking + tables. (#111) +- Added `--vram-headroom` and `--ram-budget` so users can avoid edge VRAM fits + and cap partial-offload planning to available or fixed system RAM. + +### Changed + +- Speed color now reflects practical generation speed. `~` and `?` remain + estimate-confidence markers instead of being the primary speed color. + +## [0.5.11] - 2026-06-18 + +### Added + +- Multi-GPU simulation for repeated `--gpu` flags, comma-separated GPU specs, + and count shorthand such as `2x RTX 4090`. The fit model uses a conservative + effective VRAM budget and keeps speed confidence low for split-device + recommendations. (#113) +- `python -m whichllm` now runs the CLI entrypoint. (#116) +- `--gpu-only` and `--fit full-gpu` now filter recommendations to models that + fit fully in GPU VRAM. `--fit any` keeps the existing behavior. (#119, #122) +- T5 lineage support so T5-family models get version-aware benchmark handling. + +### Fixed + +- Fixed UTF-8 decoding for cached model and benchmark data on systems whose + default filesystem encoding is not UTF-8. (#121) +- GTX 1650 simulation now distinguishes GDDR5 and GDDR6 variants by memory + clock instead of treating every card as the slower 128 GB/s model. (#115) +- RAM reserve logic now uses a bounded reserve formula instead of a fixed 80% + usable-RAM cap, which avoids underestimating machines with more system + memory. (#103) + +## [0.5.10] - 2026-06-11 + +### Fixed + +- Strong partial-offload candidates are no longer buried below weaker full-GPU + models because the final ranking sort no longer counts full-GPU fit a second + time after runtime-fit and speed penalties have already been applied. Light + partial offload is penalized less aggressively, while heavy dense offload + remains strongly discounted. (#105, #108) +- MoE partial-offload scoring now uses the active parameter working set when it + can plausibly stay on GPU, so active-small MoE models are not penalized like + dense models with the same total parameter count. (#105, #108) + +## [0.5.9] - 2026-06-10 + +### Added + +- MXFP4 and NVFP4 4-bit quantization support across ID/filename parsing, VRAM + estimation, quality penalties, speed efficiency, and family grouping. + Repos shipping these formats were previously labeled FP16 and their VRAM + requirement overestimated about 3.5x. (#99) +- Apple M5-family entries for `--gpu` simulation. (#92) +- Kepler-era Quadro bandwidth and compute capability entries. (#75) + +### Fixed + +- AMD discrete GPU detection on Linux: rocm-smi names are read from the + correct `Card Series` key, compound lspci names such as + `Navi 22 [Radeon RX 6700/6700 XT/6750 XT ...]` resolve bandwidth, sysfs VRAM + enriches the fallback path, and discrete cards are no longer mislabeled + `shared memory`. Adds RX 6750 XT / RX 6700 / RX 6650 XT / RX 6600 series and + Radeon AI PRO R9700 to the bandwidth catalog. (#61, #68) +- Community GGUF repos without `base_model` metadata (for example + `unsloth/...-GGUF`) now inherit the official model's benchmark score by + name matching instead of falling through to no evidence. (#94) +- GPU bandwidth detection no longer depends solely on the hand-curated + catalog. When a detected card is missing from `GPU_BANDWIDTH`, bandwidth is + now resolved from the bundled TechPowerUp database (dbgpu, 2824 GPUs) using + strict name matching only: an exact normalized hit or a name plus VRAM-size + bin, never fuzzy. Laptop / Mobile / Max-Q names can no longer inherit a + desktop card's bandwidth, and VRAM bins written without a space + (`RTX A2000 12GB`) are recognized. This fixes the cluster of reports where + an uncatalogued GPU showed `BW: N/A`, was estimated at `0.0 tok/s`, and + received oversized recommendations (#74, #98). +- Artificial Analysis Intelligence Index is fetched live again. The + artificialanalysis.ai leaderboard migrated to the Next.js App Router and no + longer ships a `__NEXT_DATA__` blob, so every run logged + `AA Index fetch failed ... __NEXT_DATA__ payload not found` and silently used + the frozen snapshot. The scraper now parses the App Router RSC stream + (`self.__next_f.push(...)`), canonicalizes AA's variant-suffixed display + names (`(Reasoning)`, `(high)`, ...) for mapping, and overlays live scores on + top of the curated fallback so a successful fetch can only add coverage. The + legacy `__NEXT_DATA__` path is kept as a secondary fallback. + +## [0.5.8] - 2026-06-05 + +### Added + +- `--context-length` now accepts shorthand values such as `64k` and `128k`. +- JSON ranking output now includes benchmark source and confidence metadata. +- Asahi Linux / Apple Silicon detection now recognizes Apple CPU and GPU names. +- Added GPU catalog coverage for `NVIDIA RTX A3000 Laptop GPU`, `RTX 3050`, + `RTX 5060`, `RTX 5070 Ti`, `RX 9070`, and `RX 9070 XT`. + +### Fixed + +- A3000 Laptop 6GB systems no longer get `0.0 tok/s` / heavy partial-offload + recommendations at the top just because bandwidth was missing. +- Windows CPU detection now falls back through PowerShell/CIM when `wmic` does + not return a useful CPU name. +- Models that cannot hold the requested context are demoted instead of staying + near the top of the ranking. +- Hugging Face and benchmark fetches now retry transient failures such as 429s + before falling back or failing. +- `Error fetching models:` now includes useful detail even when the underlying + network exception message is empty. +- Upgrade tables now show `0 GB` VRAM instead of treating zero as missing. + +### Changed + +- Curated registry data was split out of `constants.py` into + `whichllm.data.*` modules. +- Troubleshooting and cache documentation now better explain disk-cache paths + and stale fetch behavior. + +## [0.5.7] - 2026-05-20 + +### Added + +- LiveBench fallback data is now kept inline so benchmark scoring remains + available without relying on a generated sidecar file. + +### Fixed + +- DGX Spark / NVIDIA GB10 is now detected as a shared-memory NVIDIA GPU when + NVIDIA reports `memory.total` as unavailable. +- `whichllm run` now provides a Transformers `offload_folder`, avoiding crashes + when large models need disk offload. +- Cache paths now respect `XDG_CACHE_HOME`, including ignoring relative values + per the XDG base directory specification. +- Apple Silicon is now treated as shared memory in fit detection. +- Benchmark score fetching now runs concurrently. + +## [0.5.6] - 2026-05-18 + +### Added + +- Speed estimates now include confidence metadata and an estimated tok/s range + in table and JSON output, so uncertain backend/model predictions are visible. +- Windows now has an AMD/Intel GPU detection fallback via + `Win32_VideoController`, including 64-bit registry memory reads for GPUs + where `AdapterRAM` is capped around 4 GB. + +### Fixed + +- MoE speed estimates now use active-parameter metadata and a + bandwidth-scaled read floor, improving shared-memory APU estimates without + over-promoting sparse models on high-bandwidth GPUs. +- Newer MoE model metadata now recognizes A3B-style active-parameter names. +- Ryzen AI / Radeon 890M-class Windows iGPUs are modeled as shared-memory AMD + GPUs instead of CPU-only or tiny-VRAM discrete GPUs. +- Mixed dedicated-GPU plus shared-memory-iGPU systems no longer sum unrelated + memory pools as one full-GPU target. +- Windows AMD GPUs no longer receive a misleading ROCm-only warning when + Vulkan or DirectML backends may be valid. + +## [0.5.5] - 2026-05-17 + +### Fixed + +- `whichllm run` now resolves auto-picked GGUF recommendations to a real + GGUF repository and file before launch, instead of falling back to the + official Transformers repository. This fixes the accidental Transformers path + for models such as `Qwen/Qwen3.6-27B`. + +## [0.5.4] - 2026-05-17 + +### Fixed + +- Strix Halo / Ryzen AI MAX systems are now modeled as AMD shared-memory APUs + instead of tiny-VRAM discrete GPUs. `STRXLGEN`, `Radeon 8050S`, + `Radeon 8060S`, and related names get a 256 GB/s bandwidth estimate and use + the system shared-memory pool for fit checks, avoiding false CPU-only, + 99%-offload, and `0 tok/s` recommendations. + +## [0.5.3] - 2026-05-17 + +### Added + +- Linux Intel integrated GPU detection via `/sys/class/drm`, so Intel iGPU + systems are no longer always treated as CPU-only. +- NVIDIA `nvidia-smi` fallback detection when pynvml is missing, NVML init + fails, or NVML reports no devices. +- Apple-prefixed Apple Silicon simulator aliases, so `--gpu "Apple M3 Max"` + resolves the same way as `--gpu "M3 Max"`. + +### Fixed + +- `whichllm run` transformers chat generation now passes tokenizer mappings + into `model.generate(**inputs)`, fixing the `KeyError: 'shape'` crash path. +- RTX 5060 Ti bandwidth lookup now reports 448 GB/s instead of `N/A`. + +### Changed + +- README install guidance now prefers `uvx` / `uv tool install`. +- Removed the old marketing note from the repository and added sponsor metadata. + +## [0.5.2] - 2026-05-15 + +### Added + +- Curated vision-language benchmark source (`benchmark_sources/vision.py`): + a 0-100 multimodal capability index (MMMU-Pro / MMBench / general + multimodal, 2026-05) covering the Qwen3-VL / Qwen2.5-VL / Qwen2-VL / + Llama-Vision / Phi-vision / Gemma-3 / Pixtral / InternVL3 lines. +- Benchmark snapshot date is now shown under every ranking so a stale + recommendation is self-evident instead of silently trusted. +- Round 3 regression suite (`tests/test_r3_regressions.py`, 20 tests), + each verified to fail when its fix is reverted. + +### Fixed + +- `--profile vision` generation inversion: text leaderboards do not + score VLMs, so the only model with a direct hit was a + two-generations-old Qwen2-VL-7B, which outranked the current + Qwen3-VL-32B even on an 80 GB GPU. Vision models now score from the + curated multimodal index (Qwen3-VL-32B leads at 73-76). +- Apple Silicon partial-offload speed was estimated ~3x too low: the + flat 0.45x PCIe penalty was applied to unified memory, where spilled + weights stay in the same high-bandwidth pool. DeepSeek-R1-class + models on M2/M3 Ultra now report a realistic 4-15 t/s instead of + ~1.7. Discrete GPUs keep the 0.45x penalty. +- Duplicate `Qwen/Qwen3-Coder-30B-A3B-Instruct` key in the LiveBench + fallback (silently scored 62 instead of the intended 58, and broke + CI lint via ruff F601). +- `ruff format` / `ruff check` are now clean across the codebase, so + the Lint CI job passes (it was red for the entire 0.5.1 release). + +### CI + +- GitHub Actions updated to the Node 24 runtime (`checkout@v5`, + `setup-python@v6`); the Node 20 actions are deprecated from 2026-06. + +## [0.5.1] - 2026-05-14 + +### Added + +- `whichllm upgrade` subcommand: side-by-side comparison of the current + machine against potential GPU upgrades, with a verdict (worth it / + meaningful / marginal / flat / downgrade). +- Apple Silicon support in `--gpu` flag (M1-M4 base / Pro / Max / Ultra) + so simulator runs no longer fuzzy-match to ATI Rage Mobility-M1 and + emit spurious AMD ROCm warnings. +- Curated LiveBench, Arena AA, and Aider benchmark source modules with + frozen 2026-Q2 fallbacks for offline operation. +- Curated entries for reasoning / thinking lines: `Qwen/QwQ-32B`, + `Qwen3-4B-Thinking-2507`, `DeepSeek-R1-Distill-Qwen-32B/14B` and + `Llama-8B`. +- Frontier-model surfacing for 2026-Q2 releases that do not auto-surface + via cardinality (Kimi-K2, MiMo, DeepSeek-V4, GLM-5, Qwen3.6/Next, + gpt-oss, Llama-4, Mistral Small/Large, Devstral, Codestral, MiniMax, + Granite 3.3/4.0, Olmo-3, Nemotron-3). +- VRAM-aware auto floor for `--profile general` so tiny GPUs surface + full-GPU 3-4B picks instead of partial-offload-only 7B+. + +### Changed + +- VRAM estimation: KV cache scaled to 3.5 MB / billion-param / Kctx (was + 0.5 MB) so 128K contexts are realistic; MoE KV uses active*4 to model + attention head sharing; activation overhead refined. +- Speed estimation: per-quant efficiency table, per-backend multiplier + (CUDA 1.0, Apple 0.82, AMD 0.78, Intel 0.65), MoE active-ratio floor, + partial-offload penalty. +- Ranking: composite family selection key replaces tier dominance; + size_score cap 20 → 35; MoE size_score uses total params; + `_knowledge_capacity_b` so `--min-params` no longer hides + Qwen3-Next-80B-A3B on its 3B active. +- Benchmark merging splits frozen (OLLB v2, Arena ELO) from current + (AA, LiveBench, Aider) with separate caps and lineage-aware recency + demotion so stale 2024-era leaderboards stop over-rewarding older + generations. +- httpx `AsyncClient` uses `follow_redirects=True` so case-mismatch HF + URLs (307) no longer silently drop frontier IDs. + +### Fixed + +- Reject benchmark inheritance when actual params differ by more than + 2x from the family's dominant member, catching draft/MTP/abliterated + forks that share a `family_id` with their much larger base + (e.g. a 6.6B "imatrix-aligned" inheriting from a 158B base). +- Family grouping prefers the upstream-referenced model as the family + base instead of the highest-downloads member, so a popular fork no + longer overrides the official base for `family_id` assignment. +- MoE active-parameter registry corrected (gpt-oss-20b 3.6B, + gpt-oss-120b 5.1B, MiniMax-M2 10B). +- Quality floor (≥ 20) and speed floor (≥ 1.5 t/s) drop junk Q1_0 / + Bonsai-class attack vectors. +- 11 non-existent HF IDs removed from curated fallbacks (Kimi K2.5/K2.6, + GLM-5-Turbo, OLMo-3-32B, Llama-3.2-8B, Codestral-25.08, Mistral-Large-3 + etc.). + +## [0.4.0] - 2026-03-09 + +### Added + +- `whichllm plan` subcommand — reverse lookup to find what GPU you need for a model +- Ollama integration examples and shell alias +- Homebrew formula for `brew install whichllm` +- VHS tape file for recording CLI demo GIF +- GitHub Actions CI/CD (tests, lint, PyPI publish) +- CONTRIBUTING.md, CODE_OF_CONDUCT.md +- Issue and PR templates +- PyPI metadata (classifiers, keywords, URLs) + +## [0.3.0] - 2026-03-09 + +### Added + +- Evidence filtering options (`--evidence`, `--direct`) in CLI and ranking logic +- A100/H100 80GB aliases to GPU simulator +- Eval benchmark integration with confidence-based score dampening +- BenchmarkEvidence with confidence-aware size interpolation +- HuggingFace evalResults as supplementary benchmark source + +## [0.2.2] + +### Added + +- `--version` option to display package version + +### Changed + +- Updated demo image asset + +## [0.2.1] + +### Added + +- Vision model support based on task profile (`--profile vision`) + +## [0.2.0] + +### Added + +- `--status` flag to show Speed/Fit columns in output +- Published date and download count columns in display +- `published_at` backfill for ranking display +- GGUF-only backend filtering for model ranking +- Task profile support (`--profile`) for general, coding, vision, math +- GPU simulation (`--gpu`, `--vram`) for testing different hardware +- JSON output mode (`--json`) +- Rich table output with color-coded scores +- GPU detection for NVIDIA, AMD, and Apple Silicon +- HuggingFace API integration for model fetching +- Quantization-aware VRAM calculation +- Cache system with TTL (6h models, 24h benchmarks) + +## [0.1.0] + +### Added + +- Initial release +- Basic hardware detection +- Simple model ranking with Typer CLI diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3424433 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,39 @@ +# Code of Conduct + +## Our Pledge + +We are committed to making participation in this project a welcoming experience for everyone. + +## Our Standards + +Examples of positive behavior: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints +- Gracefully accepting constructive feedback +- Focusing on what is best for the community +- Owning your contributions, including code written with help from AI tools + +Examples of unacceptable behavior: + +- Trolling, insulting, or derogatory comments +- Personal or political attacks +- Publishing others' private information without permission + +## AI-assisted Contributions + +AI tools are welcome here. The rule is simple: make it work, understand what it +does, and own the result. + +If you submit code, docs, tests, or examples, you are responsible for them +whether you typed every line yourself or asked a tool to help. Review the work, +test the parts that matter, and respond to feedback or bugs as you would for +hand-written work. + +## Enforcement + +Instances of unacceptable behavior may be reported by opening a GitHub issue or contacting the maintainer. All reports will be reviewed and investigated. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2867b88 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,55 @@ +# Contributing to whichllm + +Thanks for your interest in contributing! Here's how you can help. + +## Development Setup + +```bash +git clone https://github.com/Andyyyy64/whichllm.git +cd whichllm +uv sync --dev +``` + +## Running Tests + +```bash +uv run pytest +``` + +## How to Contribute + +### Bug Reports +Open an issue with: +- Your hardware (GPU model, VRAM, OS) +- Python version +- Steps to reproduce +- Expected vs actual behavior + +### Feature Requests +Open an issue describing the feature and why it would be useful. + +### Pull Requests +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/your-feature`) +3. Make your changes +4. Run tests (`uv run pytest`) +5. Submit a PR + +### AI-assisted Contributions +AI-assisted code is welcome. Use the tools that help you move faster. Working +code wins. + +The bar is practical: make it work, make it fit the project, and make it +reviewable. If you wrote it or asked a tool to write it, you own it. Read it, +test the parts that matter, and be ready to explain or fix it. + +### Adding GPU Support +To add a new GPU to the bandwidth database, edit `src/whichllm/constants.py` and add the GPU specs. + +## Code Style +- Follow existing code conventions +- Use type hints +- Add tests for new functionality + +## License +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b4416fe --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Andyyyy64 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e38d62f --- /dev/null +++ b/README.md @@ -0,0 +1,513 @@ +# whichllm + +[![PyPI version](https://img.shields.io/pypi/v/whichllm)](https://pypi.org/project/whichllm/) +[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Tests](https://github.com/Andyyyy64/whichllm/actions/workflows/test.yml/badge.svg)](https://github.com/Andyyyy64/whichllm/actions/workflows/test.yml) +[![Sponsor](https://img.shields.io/badge/Sponsor-GitHub%20Sponsors-EA4AAA?logo=githubsponsors)](https://github.com/sponsors/Andyyyy64) + +

+ Andyyyy64%2Fwhichllm | Trendshift +

+ +**Find the best local LLM that actually runs on your hardware.** + +Auto-detects your GPU/CPU/RAM and ranks the top models from HuggingFace that fit your system. + +[日本語版はこちら](docs/README.ja.md) + +## Quick start + +Run the recommendation command once, with no project setup. + +```bash +uvx whichllm@latest +``` + +Simulate a GPU before you buy hardware. + +```bash +uvx whichllm@latest --gpu "RTX 4090" +``` + +Install it when you use it often. + +```bash +uv tool install whichllm +uv tool upgrade whichllm # update an existing install +``` + +Other install paths. + +```bash +brew install andyyyy64/whichllm/whichllm +pip install whichllm +``` + +## Want a safer pick? + +By default, whichllm is ambitious. It ranks the best model that looks runnable +on your machine, including partial RAM offload and near-edge VRAM fits when +they seem usable. + +If you want a more comfortable LM Studio-style recommendation, start with: + +```bash +uvx whichllm@latest --gpu-only --speed usable --vram-headroom 1GB +``` + +This keeps only models that fit fully in GPU VRAM, filters out slow estimates, +and leaves extra VRAM for runtime overhead. + +If LM Studio still says the model is slightly too large, increase the headroom: + +```bash +uvx whichllm@latest --gpu-only --speed usable --vram-headroom 1.5GB +``` + +## Common workflows + +After install, run `whichllm` directly. For one-off runs, replace `whichllm` +with `uvx whichllm@latest`. + +```bash +# Best models for this machine +whichllm + +# Pretend you have a specific GPU +whichllm --gpu "RTX 4090" + +# Override detected iGPU/unified-memory limits +whichllm --vram 8 --ram-bandwidth 68 + +# Only show models that fit fully in GPU VRAM +whichllm --gpu-only +whichllm --fit gpu + +# Simulate a multi-GPU workstation +whichllm --gpu "2x RTX 4090" + +# Hide models that are technically runnable but too slow +whichllm --speed usable +whichllm --speed fast + +# Pasteable GitHub / Slack / Discord output +whichllm --markdown + +# Compare upgrade candidates +whichllm upgrade "RTX 4090" "RTX 5090" "H100" + +# Find the GPU needed for a model +whichllm plan "llama 3 70b" + +# Start a chat with a model +whichllm run "qwen 2.5 1.5b gguf" + +# Print copy-paste Python +whichllm snippet "qwen 7b" + +# Return JSON for scripts +whichllm --top 1 --json +``` + +![demo](assets/demo.gif) + +## See it + +```text +$ whichllm --gpu "RTX 4090" + +#1 Qwen/Qwen3.6-27B 27.8B Q5_K_M score 92.8 27 t/s +#2 Qwen/Qwen3-32B 32.0B Q4_K_M score 83.0 31 t/s +#3 Qwen/Qwen3-30B-A3B 30.0B Q5_K_M score 82.7 102 t/s +``` + +The 32B model **fits your card fine** — whichllm still ranks the 27B #1, +because it scores higher on real benchmarks and is a newer generation. +A size-only "what fits?" tool would hand you the bigger one. That gap is +the whole point of whichllm. (Note #3: a MoE model at 102 t/s — speed is +ranked on *active* params, quality on *total*.) + +## What can I run? + +Real top picks (snapshot 2026-05 — your results track **live** HuggingFace +data, this is not a static list): + +| Hardware | VRAM | Top pick | Speed | +|---|---|---|---| +| RTX 5090 | 32 GB | `Qwen3.6-27B` · Q6_K · score 94.7 | ~40 t/s | +| RTX 4090 / 3090 | 24 GB | `Qwen3.6-27B` · Q5_K_M · score 92.8 | ~27 t/s | +| RTX 4060 | 8 GB | `Qwen3-14B` · Q3_K_M · score 71.0 | ~22 t/s | +| Apple M3 Max | 36 GB | `Qwen3.6-27B` · Q5_K_M · score 89.4 | ~9 t/s | +| CPU only | — | `gpt-oss-20b` (MoE) · Q4_K_M · score 45.2 | ~6 t/s | + +`whichllm --gpu ""` simulates any of these before you buy. +By default, rankings include full-GPU, partial-offload, and CPU-only +candidates when they are usable. Use `--gpu-only` or `--fit full-gpu` when +you only want models that fit entirely in GPU VRAM. +The default table shows memory, estimated generation speed, fit type, and +published date. Speed is colored by practical usability: under 4 tok/s is red, +4-10 is yellow, 10-30 is green, and 30+ is bright green. `~` / `?` still mark +estimate confidence. + +## Why whichllm? + +Fitting a model into your VRAM is the easy part. The hard part is knowing +**which of the models that fit is actually the best** — and that is what +whichllm is built to get right. + +- **Evidence-based ranking, not a size heuristic** — The top pick is + chosen from merged real benchmarks (LiveBench, Artificial Analysis, + Aider, multimodal/vision, Chatbot Arena ELO, Open LLM Leaderboard) — + never "the biggest model that happens to fit." +- **Recency-aware** — Stale leaderboards are demoted along each model's + lineage, so a 2024 model can't outrank a current-generation one on an + outdated score. The benchmark snapshot date is printed under every + ranking, so a stale recommendation is self-evident instead of silently + trusted. +- **Evidence-graded and guarded** — Every score is tagged + `direct` / `variant` / `base` / `interpolated` / `self-reported` and + discounted by confidence. Fabricated uploader claims and cross-family + inheritance (a small fork borrowing its much larger base's score) are + actively rejected. +- **Architecture-aware estimates** — VRAM = weights + GQA KV cache + + activation + overhead; speed is bandwidth-bound with per-quant + efficiency, per-backend factors, MoE active-vs-total split, and + unified-memory vs discrete-PCIe partial-offload modeling. +- **One command, scriptable** — `whichllm` prints the answer; add + `--json | jq` for pipelines. No TUI, no keybindings to memorize. +- **Live data** — Models fetched directly from the HuggingFace API, with + curated frozen fallbacks for offline or rate-limited use. + +## Features + +- **Auto-detect hardware** — NVIDIA, AMD, Intel, Apple Silicon, CPU-only +- **Smart ranking** — Scores models by VRAM fit, speed, and benchmark quality +- **One-command chat** — `whichllm run` downloads and starts a chat session instantly +- **Code snippets** — `whichllm snippet` prints ready-to-run Python for any model +- **Live data** — Fetches models directly from HuggingFace (cached for performance) +- **Benchmark-aware** — Integrates real eval scores with confidence-based dampening +- **Task profiles** — Filter by general, coding, vision, or math use cases +- **GPU simulation** — Test with any GPU: `whichllm --gpu "RTX 4090"` +- **Multi-GPU simulation** — Repeat `--gpu`, use commas, or write `2x RTX 4090` +- **Full-GPU filter** — `--gpu-only` / `--fit full-gpu` hides offload candidates +- **Speed-aware filtering** — `--speed usable|fast` hides slow rows by threshold +- **Markdown output** — `--markdown` / `-m` prints pasteable GFM tables +- **Runtime memory budgets** — `--vram-headroom` and `--ram-budget` avoid edge fits +- **Hardware planning** — Reverse lookup: `whichllm plan "llama 3 70b"` +- **Upgrade planning** — Compare your current machine with candidate GPUs +- **JSON output** — Pipe-friendly: `whichllm --json` + +## Run & Snippet + +Try any model with a single command. No manual installs needed — whichllm +creates an isolated environment via `uv`, installs dependencies, downloads the +model, and starts an interactive chat. + +![run demo](assets/demo-run.gif) + +```bash +# Chat with a model (auto-picks the best GGUF variant) +whichllm run "qwen 2.5 1.5b gguf" + +# Auto-pick the best model for your hardware and chat +whichllm run + +# CPU-only mode +whichllm run "phi 3 mini gguf" --cpu-only +``` + +Works with **all model formats**: +- **GGUF** — via `llama-cpp-python` (lightweight, fast) +- **AWQ / GPTQ** — via `transformers` + `autoawq` / `auto-gptq` +- **FP16 / BF16** — via `transformers` + +Get a **copy-paste Python snippet** instead: + +```bash +whichllm snippet "qwen 7b" +``` + +```python +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="Qwen/Qwen2.5-7B-Instruct-GGUF", + filename="qwen2.5-7b-instruct-q4_k_m.gguf", + n_ctx=4096, + n_gpu_layers=-1, + verbose=False, +) + +output = llm.create_chat_completion( + messages=[{"role": "user", "content": "Hello!"}], +) +print(output["choices"][0]["message"]["content"]) +``` + +## Usage + +```bash +# Auto-detect hardware and show best models +whichllm + +# Simulate a GPU (e.g. planning a purchase) +whichllm --gpu "RTX 4090" +whichllm --gpu "RTX 5090" +# Specify variant +whichllm --gpu "RTX 5060 16" +# Override detected iGPU/unified-memory limits +whichllm --vram 8 --ram-bandwidth 68 +# Simulate multiple GPUs +whichllm --gpu "2x RTX 4090" +whichllm --gpu "RTX 4090" --gpu "RTX 3090" +whichllm --gpu "RTX 4090, RTX 3090" + +# Only show models that fit entirely in GPU VRAM +whichllm --gpu-only +whichllm --fit gpu +whichllm --fit full-gpu + +# Avoid edge fits and background-RAM surprises +whichllm --vram-headroom 1.5GB +whichllm --ram-budget available +whichllm --ram-budget 8GB + +# CPU-only mode +whichllm --cpu-only + +# More results / filters +whichllm --top 20 +whichllm --details # show Downloads metadata instead of runtime columns +whichllm --speed usable # minimum 10 tok/s +whichllm --speed fast # minimum 30 tok/s +whichllm --min-speed 4 # exact tok/s floor +whichllm --markdown # pasteable GitHub-Flavored Markdown table +whichllm --profile coding +whichllm --context-length 64k +whichllm --quant Q4_K_M +whichllm --min-speed 30 # exact tok/s floor +whichllm --evidence base # allow id/base-model matches +whichllm --evidence strict # id-exact only (same as --direct) +whichllm --direct + +# JSON output +whichllm --json + +# Force refresh (ignore cache) +whichllm --refresh + +# Show hardware info only +whichllm hardware + +# Plan: what GPU do I need for a specific model? +whichllm plan "llama 3 70b" +whichllm plan "Qwen2.5-72B" --quant Q8_0 +whichllm plan "mistral 7b" --context-length 32768 + +# Upgrade: compare your current machine against candidate GPUs +whichllm upgrade "RTX 4090" "RTX 5090" "H100" +whichllm upgrade "Apple M4 Max" --top 5 + +# Run: download and chat with a model instantly +whichllm run "qwen 2.5 1.5b gguf" +whichllm run # auto-pick best for your hardware + +# Snippet: print ready-to-run Python code +whichllm snippet "qwen 7b" +whichllm snippet "llama 3 8b gguf" --quant Q5_K_M +``` + +Markdown output is intended for GitHub issues, READMEs, Slack, Discord, and +blog posts: + +```bash +whichllm --markdown +whichllm -m --top 5 --gpu "RTX 4090" +``` + +JSON model rows include `fit_type`, `vram_required_bytes`, +`vram_available_bytes`, `uses_multi_gpu`, `multi_gpu_effective_vram_bytes`, +`estimated_tok_per_sec`, `speed_confidence`, `speed_range_tok_per_sec`, +`speed_notes`, `benchmark_source`, and `benchmark_confidence`. The speed range +is a planning range, not a live benchmark. + +## Integrations + +### Ollama + +Use JSON output to feed scripts that map HuggingFace IDs to your local Ollama +model names: + +```bash +# Pick the top HuggingFace model ID +whichllm --top 1 --json | jq -r '.models[0].model_id' + +# Find the best coding model ID +whichllm --profile coding --top 1 --json | jq -r '.models[0].model_id' +``` + +Ollama model names do not always match HuggingFace repo IDs, so a small mapping +step is usually needed before `ollama run`. + +### Shell alias + +Add to your `.bashrc` / `.zshrc`: + +```bash +alias bestllm='whichllm --top 1 --json | jq -r ".models[0].model_id"' +# Usage: ollama run $(bestllm) +``` + +## Scoring + +Each model gets a 0-100 score. Benchmark quality and size form the core; +evidence confidence and runtime fit then scale it, with speed, source +trust, and popularity as adjustments. + +| Factor | Effect | Description | +|--------|--------|-------------| +| Benchmark quality | core | Merged LiveBench / Artificial Analysis / Aider / Vision / Arena ELO / Open LLM Leaderboard, weighted by source confidence | +| Model size | up to 35 | `log2`-scaled world-knowledge proxy (MoE uses total params) | +| Quantization | × penalty | Lower-bit quants discounted multiplicatively | +| Evidence confidence | ×0.55–1.0 | none / self-reported ×0.55, inherited ×0.78, direct full | +| Runtime fit | ×0.50–1.0 | partial-offload ×0.72, CPU-only ×0.50 | +| Speed | -8 to +8 | Usability gate vs a fit-dependent tok/s floor; reported with confidence and range metadata | +| Source trust | -5 to +5 | Official-org bonus, known-repackager penalty | +| Popularity | tie-breaker | Downloads/likes; weight shrinks as evidence strengthens | + +Score markers: +- **`~`** (yellow) — No direct benchmark; score inherited/interpolated from the model family +- **`!sr`** (bright yellow) — Uploader-reported benchmark only, not independently verified +- **`?`** (red) — No benchmark data available + +Speed display: +- **red** — Slow generation speed (`<4 tok/s`) +- **yellow** — Marginal generation speed (`4-10 tok/s`) +- **green** — Usable generation speed (`10-30 tok/s`) +- **bright green** — Fast local generation speed (`>=30 tok/s`) +- **`~`** (yellow) — Estimated tok/s range is available +- **`?`** (red) — Low-confidence speed estimate; backend/runtime sensitivity is high + +## Documentation + +- [CLI reference](docs/cli.md) +- [How it works](docs/how-it-works.md) +- [Scoring](docs/scoring.md) +- [Hardware detection and simulation](docs/hardware.md) +- [Run and snippet](docs/run-snippet.md) +- [Troubleshooting](docs/troubleshooting.md) + +## How it works + +### Data pipeline + +1. **Model fetching** — Fetches popular models from HuggingFace API: + - Text-generation (downloads + recently updated) + - GGUF-filtered (separate query for coverage) + - Vision models (`image-text-to-text`) when `--profile vision` or `any` +2. **Benchmark sources** — *Current tier* (LiveBench, Artificial Analysis + Index, Aider) merged live when reachable, plus a curated multimodal / + vision index; *frozen tier* (Open LLM Leaderboard v2, Chatbot Arena + ELO). Tiers have separate caps and lineage-aware recency demotion so + stale leaderboards stop over-rewarding older generations. +3. **Benchmark evidence** — Five resolution levels, increasingly discounted: + - `direct` — Exact model ID match + - `variant` — Suffix-stripped or -Instruct variant + - `base_model` — Base model from cardData + - `line_interp` — Size-aware interpolation within model family + - `self_reported` — Uploader-claimed eval (heavily discounted) + + Inheritance is rejected when a model's params diverge more than 2× from + its family's dominant member, catching draft / MTP / abliterated forks + that share a `family_id` with a much larger base. +4. **Cache** — normally `~/.cache/whichllm/`, or `$XDG_CACHE_HOME/whichllm/` + when `XDG_CACHE_HOME` is set to an absolute path: + - `models.json` — 6h TTL + - `benchmark.json` — 24h TTL + +### Ranking engine + +1. **Hardware detection** — NVIDIA (nvidia-ml-py), AMD (ROCm/dbgpu), Intel, Apple Silicon (Metal), CPU cores, RAM, disk +2. **VRAM estimation** — Weights + KV cache + activation + framework overhead (~500MB) +3. **Compatibility** — Full GPU / Partial Offload / CPU-only; compute capability and OS checks +4. **Speed** — tok/s from GPU memory bandwidth, quantization, backend, fit type, and MoE active parameters +5. **Scoring** — Benchmark (with confidence dampening), size, quantization penalty, fit type, speed, popularity, source trust (official vs repackager) +6. **Backend filter** — Apple Silicon and CPU-only restrict to GGUF for stability; Linux+NVIDIA allows AWQ/GPTQ + +### Project structure + +``` +src/whichllm/ +├── cli.py # Typer CLI: main, plan, run, snippet, hardware +├── constants.py # Backward-compatible exports for registry data +├── data/ # GPU, quantization, framework, and lineage registries +├── hardware/ +│ ├── detector.py # Orchestrates GPU/CPU/RAM detection +│ ├── nvidia.py # NVIDIA GPU via nvidia-ml-py +│ ├── amd.py # AMD GPU (Linux) +│ ├── apple.py # Apple Silicon (Metal) +│ ├── cpu.py # CPU name, cores, AVX support +│ ├── memory.py # RAM and disk free +│ ├── gpu_simulator.py # --gpu flag: synthetic GPU from name +│ └── types.py # GPUInfo, HardwareInfo +├── models/ +│ ├── fetcher.py # HuggingFace API, model parsing, evalResults +│ ├── benchmark.py # Arena ELO, Leaderboard (parquet/rows API) +│ ├── grouper.py # Family grouping by base_model and name +│ ├── cache.py # JSON cache with TTL +│ └── types.py # ModelInfo, GGUFVariant, ModelFamily +├── engine/ +│ ├── vram.py # VRAM = weights + KV cache + activation + overhead +│ ├── compatibility.py# Fit type, disk check, compute/OS warnings +│ ├── performance.py # tok/s from bandwidth +│ ├── quantization.py # Bytes per weight, quality penalty, non-GGUF inference +│ ├── ranker.py # Scoring, evidence filter, profile/match +│ └── types.py # CompatibilityResult +└── output/ + ├── ranking.py # Rich hardware and recommendation tables + ├── json_output.py # Ranking, plan, and upgrade JSON + ├── plan.py # plan command display + ├── upgrade.py # upgrade comparison display + └── display.py # Compatibility re-export shim +``` + +## Development + +```bash +git clone https://github.com/Andyyyy64/whichllm.git +cd whichllm +uv sync --dev +uv run whichllm +uv run pytest +``` + +## Contributing + +Contributions are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +## Support + +If whichllm helped you find a model or avoid a bad hardware guess, +sponsoring is appreciated. It helps keep the project maintained: hardware +reports, packaging, test fixtures, benchmark updates, and support for more +machines. + +whichllm will stay open-source either way. Issues and PRs are always welcome. + +Useful? A GitHub star helps other people find it, and I'd genuinely like to +know what it picked for your rig. Drop it in [Issues](https://github.com/Andyyyy64/whichllm/issues). + +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=Andyyyy64/whichllm&type=Date)](https://www.star-history.com/#Andyyyy64/whichllm&Date) + +## Requirements + +- Python 3.11+ +- NVIDIA GPU detection via `nvidia-ml-py` (included by default) +- AMD / Apple Silicon detected automatically + +## License + +MIT diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..e42dfd8 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`Andyyyy64/whichllm` +- 原始仓库：https://github.com/Andyyyy64/whichllm +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/assets/demo-run.gif b/assets/demo-run.gif new file mode 100644 index 0000000..7758a66 Binary files /dev/null and b/assets/demo-run.gif differ diff --git a/assets/demo-run.tape b/assets/demo-run.tape new file mode 100644 index 0000000..e8a1a59 --- /dev/null +++ b/assets/demo-run.tape @@ -0,0 +1,45 @@ +# VHS demo recording for whichllm run/snippet +# Install: https://github.com/charmbracelet/vhs +# Run: cd assets && vhs demo-run.tape +# Output: demo-run.gif + +Output demo-run.gif + +Set FontSize 15 +Set Width 1200 +Set Height 800 +Set Theme "Catppuccin Mocha" +Set Padding 20 +Set TypingSpeed 50ms + +# Warm up cache +Hide +Type "whichllm --top 1 > /dev/null 2>&1" +Enter +Sleep 10s +Type "clear" +Enter +Sleep 500ms +Show + +# Scene 1: snippet — show the code +Type "whichllm snippet 'qwen 2.5 1.5b gguf'" +Sleep 500ms +Enter +Sleep 4s + +# Scene 2: run — download and chat +Type "whichllm run 'qwen 2.5 1.5b gguf'" +Sleep 500ms +Enter +Sleep 12s + +# Chat +Type "Explain quantum computing in one sentence." +Enter +Sleep 8s +Sleep 2s + +Type "exit" +Enter +Sleep 2s diff --git a/assets/demo.gif b/assets/demo.gif new file mode 100644 index 0000000..61be457 Binary files /dev/null and b/assets/demo.gif differ diff --git a/assets/demo.png b/assets/demo.png new file mode 100644 index 0000000..1f047ca Binary files /dev/null and b/assets/demo.png differ diff --git a/assets/demo.tape b/assets/demo.tape new file mode 100644 index 0000000..19ce3ff --- /dev/null +++ b/assets/demo.tape @@ -0,0 +1,37 @@ +# VHS demo recording for whichllm +# Install: https://github.com/charmbracelet/vhs +# Run: cd assets && vhs demo.tape +# Output: demo.gif + +Output demo.gif + +Set FontSize 15 +Set Width 1200 +Set Height 1100 +Set Theme "Catppuccin Mocha" +Set Padding 20 +Set TypingSpeed 50ms + +# Warm up cache invisibly +Hide +Type "uvx whichllm@latest --top 1 > /dev/null 2>&1" +Enter +Sleep 10s +Type "clear" +Enter +Sleep 500ms +Show + +# Scene 1: Just run it +Type "uvx whichllm@latest" +Sleep 500ms +Enter +Sleep 6s +Sleep 2s + +# Scene 2: What GPU do I need? +Type "uvx whichllm@latest plan 'qwen 72b'" +Sleep 500ms +Enter +Sleep 6s +Sleep 3s diff --git a/docs/README.ja.md b/docs/README.ja.md new file mode 100644 index 0000000..a2b422f --- /dev/null +++ b/docs/README.ja.md @@ -0,0 +1,249 @@ +# whichllm + +**手元のハードウェアで実際に動くローカルLLMを探すCLIです。** + +whichllm は GPU / CPU / RAM / ディスクを検出し、HuggingFace 上のモデルを +取得して、実行できる候補をランキングします。単に「VRAMに入る最大モデル」を +選ぶのではなく、ベンチマーク、量子化、速度、実行形態、モデル世代をまとめて +評価します。 + +[English version](../README.md) + +![demo](../assets/demo.gif) + +## インストール + +### uv + +一度だけ試す場合: + +```bash +uvx whichllm@latest +``` + +継続して使う場合: + +```bash +uv tool install whichllm +uv tool upgrade whichllm # 既存インストールを更新 +``` + +### Homebrew + +```bash +brew install andyyyy64/whichllm/whichllm +``` + +### pip + +```bash +pip install whichllm +``` + +## 余裕を持って動く候補だけ見たい場合 + +whichllm のデフォルトは少し攻めた推薦です。RAMへのpartial offloadや、 +VRAMぎりぎりの候補も、動きそうならランキングに入れます。 + +LM Studioなどで余裕を持って動かしたい場合は、まずこれを使ってください。 + +```bash +uvx whichllm@latest --gpu-only --speed usable --vram-headroom 1GB +``` + +GPUのVRAMに全部載る候補だけに絞り、遅い推定速度の候補を外し、実行時の +余白も1GB残します。 + +それでもLM Studio側で少しはみ出す場合は、余白を増やします。 + +```bash +uvx whichllm@latest --gpu-only --speed usable --vram-headroom 1.5GB +``` + +### 開発用 + +```bash +git clone https://github.com/Andyyyy64/whichllm.git +cd whichllm +uv sync --dev +uv run whichllm +uv run pytest +``` + +## まず使う + +```bash +# 自動検出しておすすめモデルを表示 +whichllm + +# GPUをシミュレートする +whichllm --gpu "RTX 4090" +whichllm --gpu "Apple M3 Max" + +# 複数GPUをシミュレートする +whichllm --gpu "2x RTX 4090" +whichllm --gpu "RTX 4090" --gpu "RTX 3090" + +# GPUのVRAMに全部載る候補だけを見る +whichllm --gpu-only +whichllm --fit gpu + +# 速度の最低ラインを指定する +whichllm --speed usable +whichllm --speed fast +whichllm --min-speed 4 + +# GitHubやSlackに貼りやすいMarkdown表で出力する +whichllm --markdown + +# 実行時のメモリ余白やRAM使用量を指定する +whichllm --vram-headroom 1.5GB +whichllm --ram-budget available + +# CPUのみとして評価する +whichllm --cpu-only + +# JSONで出力する +whichllm --json +``` + +JSONの各モデルには `estimated_tok_per_sec` に加えて、`fit_type`、 +`vram_required_bytes`、`vram_available_bytes`、`uses_multi_gpu`、 +`multi_gpu_effective_vram_bytes`、`speed_confidence`、 +`speed_range_tok_per_sec`、`speed_notes`、`benchmark_source`、 +`benchmark_confidence` が入ります。 +速度は実測値ではなく、ハードウェア情報とモデル情報からの推定です。 +通常の表には必要メモリ、推定生成速度、Fit種別、Published が表示されます。 +Downloads まで見たい場合は `--details` を使います。 +GitHub issue、README、Slack、Discord へ貼る場合は `--markdown` / `-m` +でMarkdown表として出力できます。 + +## 主なコマンド + +```bash +# 推薦ランキング +whichllm --top 20 +whichllm --quant Q4_K_M +whichllm --min-speed 30 +whichllm --speed usable +whichllm --speed fast +whichllm --markdown +whichllm --profile coding +whichllm --context-length 64k +whichllm --details +whichllm --gpu-only + +# ベンチ根拠の厳しさ +whichllm --evidence strict +whichllm --evidence base +whichllm --direct + +# モデルから必要GPUを逆算 +whichllm plan "llama 3 70b" +whichllm plan "Qwen2.5-72B" --quant Q8_0 +whichllm plan "mistral 7b" --context-length 32768 + +# 今のマシンと購入候補GPUを比較 +whichllm upgrade "RTX 4090" "RTX 5090" "H100" + +# モデルをダウンロードしてチャット +whichllm run "qwen 2.5 1.5b gguf" +whichllm run + +# 実行用Pythonコードを表示 +whichllm snippet "qwen 7b" + +# ハードウェア情報だけ表示 +whichllm hardware +``` + +## スコアの見方 + +各モデルには 0 から 100 のスコアが付きます。中心になるのはベンチマークと +モデルサイズですが、実行時に遅すぎる候補や、CPUオフロードが大きい候補は +下がります。 + +| 要素 | 役割 | +| --- | --- | +| ベンチマーク | LiveBench、Artificial Analysis、Aider、Vision、Arena、Open LLM Leaderboard を統合 | +| モデルサイズ | 知識量の近似。MoEは総パラメータを使う | +| 量子化 | Q4 / Q5 / Q6 / Q8 などの品質低下を反映 | +| 実行形態 | Full GPU、Partial Offload、CPU-only を区別 | +| 速度 | tok/s が実用ラインを下回ると減点。表示時は推定の信頼度と幅も出す | +| 根拠の強さ | direct、base_model、variant、line_interp、self_reported を区別 | +| 世代補正 | 古い凍結ベンチだけで新世代を上回らないよう調整 | + +スコア横のマーカー: + +- `~`: 直接ベンチではなく、系列や派生から推定したスコア +- `!sr`: アップローダー自己申告の評価値だけに基づくスコア +- `?`: 利用できるベンチマーク根拠がないスコア + +速度表示: + +- 赤: `4 tok/s` 未満の遅い生成速度 +- 黄: `4-10 tok/s` のぎりぎり使える生成速度 +- 緑: `10-30 tok/s` の実用的な生成速度 +- 明るい緑: `30 tok/s` 以上の高速なローカル生成速度 +- `~`: 速度推定の幅がある通常の推定値 +- `?`: backend や runtime の影響が大きい低信頼の推定値 + +## 仕組み + +1. ハードウェアを検出します。NVIDIA、AMD、Intel、Apple Silicon、CPU、RAM、 + ディスク空き容量を見ます。 +2. HuggingFace APIからモデルを取得します。人気モデル、GGUF、最近更新された + GGUF、trending、重要な frontier モデルを組み合わせます。 +3. ベンチマークを読み込みます。現在系の LiveBench / Artificial Analysis / + Aider / Vision と、凍結系の Arena / Open LLM Leaderboard を分けて扱います。 +4. `base_model` とモデル名からファミリーを作り、同じモデルの派生やGGUFを束ねます。 +5. 候補ごとに VRAM、互換性、速度、速度推定の信頼度、スコアを計算します。 +6. ファミリーごとに最も良い候補を残して表示します。 + +通常は full GPU、partial offload、CPU-only の候補をまとめて見ます。GPUの +VRAMに全部載るモデルだけを見たい場合は `--gpu-only` か +`--fit gpu` を使います。遅い候補を最初から除外したい場合は +`--speed usable`、`--speed fast` を使います。 +それぞれ `10 tok/s`、`30 tok/s` が最低ラインです。 +もっと低いラインを指定したい場合は `--min-speed 4` のように数値で指定します。 + +キャッシュは通常 `~/.cache/whichllm/` に保存されます。`XDG_CACHE_HOME` が +絶対パスで設定されている場合は、その配下の `whichllm/` を使います。 + +- `models.json`: 6時間 +- `benchmark.json`: 24時間 + +## プロジェクト構成 + +```text +src/whichllm/ +├── cli.py # Typer CLI: main, plan, upgrade, run, snippet, hardware +├── constants.py # 互換用のregistry再export +├── data/ # GPU、量子化、framework、lineageのregistry +├── hardware/ # ハードウェア検出とGPUシミュレーション +├── models/ # HuggingFace取得、ベンチ、キャッシュ、グルーピング +├── engine/ # VRAM、互換性、速度、ランキング +└── output/ # Rich表示、JSON、plan/upgrade表示 +``` + +## 詳細ドキュメント + +- [CLIリファレンス](cli.md) +- [仕組み](how-it-works.md) +- [スコアリング](scoring.md) +- [ハードウェア検出とシミュレーション](hardware.md) +- [run と snippet](run-snippet.md) +- [トラブルシュート](troubleshooting.md) + +## 動作環境 + +- Python 3.11+ +- NVIDIA GPU検出は `nvidia-ml-py` と `nvidia-smi` fallback +- AMD GPU検出は Linux / ROCm / sysfs / lspci と Windows fallback +- Intel GPU検出は Linux / sysfs / lspci と Windows fallback +- Strix Halo、Ryzen AI MAX、Radeon 890M 系は shared memory APU として扱う +- Apple Silicon検出は macOS / `system_profiler` + +## ライセンス + +MIT diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..6f95e65 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,270 @@ +# CLI reference + +This page describes the commands exposed by `whichllm`. It is based on the +Typer entrypoint in `src/whichllm/cli.py`. + +## Main command + +```bash +whichllm [OPTIONS] +``` + +Detects the current machine, loads model and benchmark data, ranks compatible +models, and prints a table. + +Common options: + +| Option | Meaning | +| --- | --- | +| `--top`, `-n` | Number of ranked models to show. Default: `10` | +| `--context-length`, `-c` | Context length used for KV cache estimation. Accepts integers or `k` shorthand such as `64k`. Default: `4096` | +| `--quant`, `-q` | Keep only a quantization type such as `Q4_K_M` | +| `--min-speed` | Keep only models above an exact tok/s estimate | +| `--speed` | Named speed floor: `any`, `usable` (`10 tok/s`), or `fast` (`30 tok/s`) | +| `--fit` | Runtime fit filter: `any`, `gpu`, or `full-gpu` | +| `--gpu-only` | Alias for `--fit full-gpu`; excludes partial offload and CPU-only candidates | +| `--profile` | Ranking profile: `general`, `coding`, `vision`, `math`, `any` | +| `--evidence` | Benchmark evidence filter: `strict`, `base`, `any` | +| `--direct` | Alias for `--evidence strict` | +| `--status` | Compatibility option. Runtime columns are now shown by default | +| `--details` | Show download metadata instead of runtime columns | +| `--min-params` | Minimum model knowledge capacity in billions of parameters | +| `--json` | Print machine-readable JSON | +| `--markdown`, `-m` | Print a pasteable GitHub-Flavored Markdown table | +| `--refresh` | Ignore caches and fetch models/benchmarks again | +| `--cpu-only` | Ignore GPUs and rank for CPU-only use | +| `--gpu` | Simulate GPU(s) by name. Accepts repeated flags, comma-separated values, and count shorthand | +| `--vram` | Override simulated GPU VRAM or detected GPU usable VRAM in GB | +| `--bandwidth`, `--ram-bandwidth` | Override GPU/RAM bandwidth in GB/s | +| `--gpu-index` | Detected GPU index to override when multiple GPUs are present | +| `--vram-headroom` | Reserve per-GPU memory for runtime overhead. Default: `auto`. Accepts `none`, byte values like `1.5GB`, or percentages like `10%` | +| `--ram-budget` | Cap RAM available for partial offload. Accepts `available`, byte values like `8GB`, or percentages like `50%` | +| `--version` | Print the installed package version | + +Environment variables: + +| Variable | Meaning | +| --- | --- | +| `HF_ENDPOINT` | Hugging Face endpoint root used for whichllm's own model metadata API calls. Example: `https://huggingface.co` or a compatible mirror root | + +`--fit any` is the default. It can include full-GPU, partial-offload, and +CPU-only candidates when they are runnable. `--fit gpu`, `--fit full-gpu`, and +`--gpu-only` keep only rows whose `fit_type` is `full_gpu`. + +The default table shows memory required, estimated generation speed, fit type, +and published date. Use `--details` when you want download counts instead. +Speed colors are absolute usability hints: red is under `4 tok/s`, yellow is +`4-10 tok/s`, green is `10-30 tok/s`, and bright green is `30+ tok/s`. The `~` +and `?` markers still refer to estimate confidence, not speed quality. + +`--vram-headroom auto` subtracts a small budget from each GPU before fit +checks, so near-edge recommendations are less likely to overflow in tools such +as LM Studio. Use `--vram-headroom none` to restore the raw detected VRAM. +`--ram-budget available` caps offload planning to current available RAM. +For detected iGPU or unified-memory systems, use `--vram` and +`--bandwidth` / `--ram-bandwidth` to override the automatically detected +usable memory and bandwidth. If multiple GPUs are detected, add `--gpu-index` +with the GPU number from `whichllm hardware`. + +Examples: + +```bash +whichllm +whichllm --gpu "RTX 4090" +whichllm --gpu "RTX 5060 Ti" --vram 16 +whichllm --vram 8 --ram-bandwidth 68 +whichllm --gpu-index 1 --vram 8 --bandwidth 68 +whichllm --gpu "2x RTX 4090" +whichllm --gpu "RTX 4090" --gpu "RTX 3090" +whichllm --gpu "RTX 4090, RTX 3090" +whichllm --profile coding --top 5 +whichllm --context-length 64k +whichllm --gpu-only +whichllm --fit gpu +whichllm --speed usable +whichllm --speed fast +whichllm --min-speed 4 +whichllm --markdown +whichllm --vram-headroom 1.5GB +whichllm --ram-budget available +whichllm --details +whichllm --evidence strict +whichllm --json | jq '.models[0]' +``` + +`--markdown` is mutually exclusive with `--json`. It prints a plain Markdown +table without the Rich hardware panel, colors, or box-drawing characters. + +Ranking JSON model rows include: + +| Field | Meaning | +| --- | --- | +| `fit_type` | Runtime fit classification: `full_gpu`, `partial_offload`, or `cpu_only` | +| `vram_required_bytes` | Estimated runtime memory requirement for the candidate | +| `vram_available_bytes` | GPU memory budget used for the fit check | +| `uses_multi_gpu` | Whether the fit check used more than one GPU | +| `multi_gpu_effective_vram_bytes` | Conservative effective VRAM budget for multi-GPU fits, when applicable | +| `estimated_tok_per_sec` | Point estimate used by ranking | +| `speed_confidence` | `high`, `medium`, or `low` | +| `speed_range_tok_per_sec` | Estimated lower/upper tok/s range, when available | +| `speed_notes` | Short reasons for the confidence level | +| `benchmark_status` | Display marker category for benchmark evidence | +| `benchmark_source` | How benchmark evidence was matched: `direct`, `variant`, `base_model`, `line_interp`, `self_reported`, or `none` | +| `benchmark_confidence` | Confidence in the benchmark match, `0.0`–`1.0` | + +The top-level `hardware` object also includes `usable_vram_bytes` per GPU, +`ram_budget_bytes`, and `budget_notes` when memory budgets are active. + +## `hardware` + +```bash +whichllm hardware [OPTIONS] +``` + +Prints detected hardware without ranking models. The same simulation flags are +available here: + +```bash +whichllm hardware +whichllm hardware --cpu-only +whichllm hardware --gpu "Apple M3 Max" +whichllm hardware --gpu "RTX 3060" --vram 12 +whichllm hardware --vram 8 --bandwidth 68 +whichllm hardware --gpu "4x RTX 4090" +``` + +## `plan` + +```bash +whichllm plan MODEL_NAME [OPTIONS] +``` + +Searches for a model by HuggingFace repo ID or fuzzy terms, then estimates the +VRAM required for several quantization levels and common GPUs. + +Options: + +| Option | Meaning | +| --- | --- | +| `--context-length`, `-c` | Context length for the memory estimate. Accepts integers or `k` shorthand such as `128k`. Default: `4096` | +| `--quant`, `-q` | Target quantization. Default: `Q4_K_M` | +| `--json` | Print the plan as JSON | +| `--refresh` | Ignore model cache and fetch again | + +Examples: + +```bash +whichllm plan "llama 3 70b" +whichllm plan "Qwen2.5-72B" --quant Q8_0 +whichllm plan "mistral 7b" --context-length 32768 +whichllm plan "mistral 7b" --context-length 32k +``` + +## `upgrade` + +```bash +whichllm upgrade TARGET_GPUS... [OPTIONS] +``` + +Compares the current machine against one or more simulated GPUs. The CPU, RAM, +disk, and OS come from the current machine; only the GPU changes. + +Options: + +| Option | Meaning | +| --- | --- | +| `--context-length`, `-c` | Context length used for ranking. Accepts integers or `k` shorthand such as `64k`. Default: `8192` | +| `--top`, `-n` | Best-N models to compare per GPU. Default: `3` | +| `--profile` | Ranking profile. Default: `general` | +| `--cpu-only` | Use CPU-only as the current baseline | +| `--json` | Print comparison JSON | +| `--refresh` | Ignore caches and fetch again | + +Examples: + +```bash +whichllm upgrade "RTX 4090" "RTX 5090" "H100" +whichllm upgrade "Apple M4 Max" --top 5 +whichllm upgrade "RX 7900 XTX" --profile coding +whichllm upgrade "RTX 4090" --context-length 128k +``` + +## `run` + +```bash +whichllm run [MODEL_NAME] [OPTIONS] +``` + +Creates a temporary Python script, launches it through `uv run --no-project`, +installs the needed inference packages into that isolated run, and starts an +interactive chat. + +If `MODEL_NAME` is omitted, whichllm ranks models for the current hardware and +uses the top result. + +Options: + +| Option | Meaning | +| --- | --- | +| `--context-length`, `-c` | Context length for the generated chat script. Accepts integers or `k` shorthand such as `64k` | +| `--quant`, `-q` | Preferred GGUF quantization | +| `--refresh` | Ignore model cache and fetch again | +| `--cpu-only` | Force CPU-only execution in the generated script | + +Examples: + +```bash +whichllm run +whichllm run "qwen 2.5 1.5b gguf" +whichllm run "phi 3 mini gguf" --cpu-only +whichllm run "mistral 7b gguf" --context-length 64k +``` + +`run` requires `uv` in `PATH`. + +## `snippet` + +```bash +whichllm snippet [MODEL_NAME] [OPTIONS] +``` + +Prints a ready-to-run Python snippet for the selected model. GGUF models use +`llama-cpp-python`; non-GGUF models use `transformers`. + +Options: + +| Option | Meaning | +| --- | --- | +| `--quant`, `-q` | Preferred GGUF quantization | +| `--refresh` | Ignore model cache and fetch again | + +Examples: + +```bash +whichllm snippet "qwen 7b" +whichllm snippet "llama 3 8b gguf" --quant Q5_K_M +``` + +## Evidence filters + +`--evidence` controls which benchmark matches are allowed into the ranking. + +| Mode | Allows | +| --- | --- | +| `strict` | Exact independent benchmark matches only | +| `base` | Exact, variant, and `cardData.base_model` matches | +| `any` | All evidence levels, including line interpolation and self-reported values | + +`--direct` is kept as a shorter alias for `--evidence strict`. + +## Profiles + +The ranker detects specialization from repository names. + +| Profile | Behavior | +| --- | --- | +| `general` | Excludes coding, vision, and math-specialized names | +| `coding` | Keeps coding-specialized names | +| `vision` | Keeps vision or multimodal names and includes VLM candidates | +| `math` | Keeps math-specialized names | +| `any` | Keeps all recognized model types and includes VLM candidates | diff --git a/docs/hardware.md b/docs/hardware.md new file mode 100644 index 0000000..d781eec --- /dev/null +++ b/docs/hardware.md @@ -0,0 +1,237 @@ +# Hardware detection and simulation + +whichllm detects the current machine and can also simulate hardware for +purchase planning. + +The source of truth is the `hardware/` package plus curated registry data in +`data/gpu.py`. `constants.py` remains as a compatibility export layer for older +imports. + +## Detected data + +The ranker receives a `HardwareInfo` object with: + +- GPU list +- CPU name +- physical CPU cores +- AVX2 and AVX-512 support +- total RAM +- free disk space +- OS name + +Each GPU is represented as `GPUInfo`: + +- name +- vendor +- VRAM bytes +- usable VRAM bytes, when a runtime headroom is active +- NVIDIA compute capability, when known +- CUDA or ROCm version, when known +- memory bandwidth estimate +- whether the GPU uses shared memory + +## NVIDIA + +NVIDIA detection tries `nvidia-ml-py` first. If NVML is unavailable, fails to +initialize, or returns no devices, whichllm falls back to: + +```bash +nvidia-smi --query-gpu=name,memory.total,clocks.max.memory --format=csv,noheader,nounits +``` + +If a driver rejects `clocks.max.memory`, whichllm retries the older +`name,memory.total` query. + +For known cards, curated data and strict `dbgpu` lookups provide: + +- memory bandwidth +- compute capability + +The max memory clock is used when a marketing name covers multiple memory +types. For example, GTX 1650 GDDR5 and GDDR6 cards share the same broad driver +name, so whichllm uses the reported memory clock when available and falls back +to the conservative bandwidth when it is not. + +DGX Spark / NVIDIA GB10 uses unified system memory. When the driver reports +`memory.total` as unavailable, whichllm treats GB10 as shared memory and uses +system RAM for fit checks. + +Compute capability is used to warn when a card is below the minimum expected by +common local inference tools. + +## AMD + +On Linux, AMD detection tries `rocm-smi` first: + +- product name +- VRAM +- ROCm driver version + +If `rocm-smi` is unavailable, it falls back to `lspci` and then +`/sys/class/drm`. + +On Windows, whichllm uses `Win32_VideoController` as a fallback for AMD GPUs. +When possible, it also reads the 64-bit dedicated-memory value from the +`Control\Video` registry path because `AdapterRAM` is a 32-bit field and can +cap larger cards around 4 GB. + +AMD shared-memory APUs are treated differently from discrete GPUs. Names such +as Strix Halo, Ryzen AI MAX, Radeon 8050S, Radeon 8060S, Radeon 890M, and +Radeon 780M are modeled as shared-memory systems. If the reported VRAM is just +a small aperture, whichllm uses the system memory pool for fit checks instead +of treating it as a tiny discrete GPU. + +## Intel + +Intel integrated GPUs are detected on Linux through `lspci` or sysfs, and on +Windows through `Win32_VideoController`. They do not normally report dedicated +VRAM, so whichllm records them with `0` dedicated VRAM and labels them as +shared memory. + +Discrete Intel Arc cards are kept as dedicated-memory GPUs when the device name +and memory report look like a discrete adapter. + +The Intel backend factor is lower than NVIDIA, AMD, and Apple because local LLM +GPU inference support is less mature. + +## Apple Silicon + +On macOS, whichllm uses: + +```bash +system_profiler SPHardwareDataType -json +``` + +Apple Silicon uses unified memory, so the detected chip memory is treated as +available GPU memory. Memory bandwidth is looked up by chip family when known. + +Partial offload on Apple Silicon is not penalized like discrete PCIe offload. +Weights still live in unified memory, so the speed penalty is milder. + +## CPU and memory + +CPU detection reads: + +- `/proc/cpuinfo` on Linux +- `sysctl` on macOS +- `wmic` on Windows, then PowerShell / CIM when `wmic` is unavailable or only + returns a header + +Physical core count comes from `psutil`, with a Linux `/proc/cpuinfo` fallback. + +RAM comes from `psutil.virtual_memory()`. Disk free space is checked under the +user's home directory by default. + +## GPU simulation + +Use `--gpu` to simulate a GPU: + +```bash +whichllm --gpu "RTX 4090" +whichllm hardware --gpu "Apple M3 Max" +whichllm upgrade "RTX 4090" "RTX 5090" "H100" +``` + +Simulation uses the `dbgpu` package for a TechPowerUp-backed GPU database. +whichllm adds extra handling for common aliases and Apple Silicon chips because +those are not covered by dbgpu. + +Use `--vram` when a GPU name is ambiguous, unknown, or has multiple variants: + +```bash +whichllm --gpu "RTX 5060 Ti" --vram 16 +whichllm hardware --gpu "Unknown GPU" --vram 24 +``` + +For detected integrated or unified-memory GPUs, use `--vram` and +`--bandwidth` / `--ram-bandwidth` to override the automatically detected usable +capacity and memory bandwidth: + +```bash +whichllm --vram 8 --ram-bandwidth 68 +whichllm hardware --vram 8 --bandwidth 68 +``` + +If multiple GPUs are detected, pass `--gpu-index` to choose the GPU shown by +`whichllm hardware`: + +```bash +whichllm --gpu-index 1 --vram 8 --ram-bandwidth 68 +``` + +By default, whichllm applies a small automatic VRAM headroom before fit checks. +This avoids recommending models that only fit on paper but overflow in runtimes +that need extra graph buffers or loader overhead. Tune it with: + +```bash +whichllm --vram-headroom 1.5GB +whichllm --vram-headroom 10% +whichllm --vram-headroom none +``` + +Multi-GPU simulation accepts repeated flags, comma-separated values, and count +shorthand: + +```bash +whichllm --gpu "2x RTX 4090" +whichllm --gpu "RTX 4090" --gpu "RTX 3090" +whichllm --gpu "RTX 4090, RTX 3090" +``` + +`--vram` is only supported for a single simulated GPU. For multi-GPU +simulation, use known GPU names so whichllm can resolve each card's VRAM from +the GPU database. + +## Fit types + +Compatibility checks classify a candidate into one of three fit types: + +| Fit | Meaning | +| --- | --- | +| `full_gpu` | Required memory fits in available GPU memory | +| `partial_offload` | GPU plus usable system RAM can hold the model | +| `cpu_only` | Usable system RAM can hold the model without GPU | + +If neither GPU memory nor usable RAM can hold the model, the candidate is not +ranked. + +whichllm keeps a bounded system-RAM reserve for the OS and other processes. +Use `--ram-budget available` to cap partial-offload planning to the current +available RAM reported by the OS, or pass a fixed budget such as +`--ram-budget 8GB`. + +## Multiple GPUs + +For fit checks, whichllm uses a conservative multi-GPU budget rather than +pretending all VRAM is one perfect device. It starts from raw total VRAM, applies +a small per-GPU overhead, and then applies a utilization factor. Homogeneous +sets receive a less severe reduction than heterogeneous sets. + +If a dedicated GPU is present, low-aperture shared-memory integrated GPUs are +not added to the fit pool. This avoids treating unrelated system RAM and +dedicated VRAM as one full-GPU target. + +For speed estimates, whichllm uses the largest detected GPU as the +representative device and marks multi-GPU speed as low-confidence. This avoids +claiming ideal scaling when real performance depends on backend split mode, +PCIe/NVLink bandwidth, NCCL/RCCL support, batch size, and model architecture. + +This is a practical fit approximation. It does not model every tensor-parallel +or pipeline-parallel runtime configuration. + +## Disk checks + +The compatibility check also compares estimated model weight size with free +disk space. If the model cannot be downloaded, it is marked unrunnable. + +## Known limitations + +- GPU bandwidth is a lookup or database estimate, not a live benchmark. +- Speed estimates are planning numbers. The default table and JSON fields such + as `speed_confidence` and `speed_range_tok_per_sec` show uncertainty. +- Driver, runtime, batch size, prompt length, and thermal limits can change real + performance. +- Multi-GPU runtime behavior depends on the inference backend and is only + approximated. +- Apple and shared-memory APU behavior is modeled as unified-memory style, but + real results still depend on OS pressure and memory bandwidth. diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 0000000..7ee34a3 --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,193 @@ +# How it works + +whichllm has one main job: start with the user's hardware, collect candidate +models, estimate what can run, and rank the results. + +The implementation is intentionally split into small packages: + +```text +src/whichllm/ +├── cli.py +├── constants.py +├── data/ +├── hardware/ +├── models/ +├── engine/ +└── output/ +``` + +## Request flow + +The default `whichllm` command follows this path: + +1. Validate CLI flags. +2. Detect hardware. +3. Load model cache or fetch from HuggingFace. +4. Load benchmark cache or fetch benchmark sources. +5. Group related model repos into families. +6. Flatten families back into rankable candidates. +7. Rank every candidate variant. +8. Backfill missing published dates for top results. +9. Print a Rich table or JSON. + +The `plan`, `upgrade`, `run`, `snippet`, and `hardware` subcommands reuse parts +of the same pipeline. + +## Hardware detection + +`hardware/detector.py` orchestrates detection. Each detector is fail-safe and +returns an empty result on failure. + +| Module | Role | +| --- | --- | +| `hardware/nvidia.py` | Uses `nvidia-ml-py`; falls back to `nvidia-smi`, including optional memory-clock data | +| `hardware/amd.py` | Uses `rocm-smi`; falls back to `lspci` and `/sys/class/drm` | +| `hardware/intel.py` | Detects Linux Intel iGPUs through `lspci` or sysfs | +| `hardware/windows.py` | Detects Windows AMD and Intel fallback GPUs through WMI and registry memory fields | +| `hardware/apple.py` | Uses `system_profiler` on macOS | +| `hardware/cpu.py` | Reads CPU name, physical cores, AVX2, and AVX-512 | +| `hardware/memory.py` | Reads RAM and disk free space | +| `hardware/gpu_simulator.py` | Builds synthetic GPUs for `--gpu` | + +The result is a `HardwareInfo` dataclass. GPUs are represented by `GPUInfo`. + +## Model fetching + +`models/fetcher.py` reads the HuggingFace API and turns responses into +`ModelInfo` objects. + +The fetcher combines several queries: + +1. Popular `text-generation` models sorted by downloads. +2. Popular GGUF repos. +3. Recently modified GGUF repos. +4. Trending text-generation repos, with and without the GGUF filter. +5. A curated list of frontier and hard-to-find model IDs. +6. Vision candidates from `image-text-to-text` when the active profile needs + them. + +For each repository, the parser extracts: + +- repo ID and display name +- parameter count +- active parameter count for known or config-detected MoE models +- architecture and context length +- license, downloads, likes, published date +- GGUF variants and file sizes +- `cardData.base_model` +- conservative HuggingFace `evalResults` values + +Parameter counts can come from safetensors metadata, GGUF metadata, config +estimation, name hints, or a curated fallback table for important models with +missing metadata. + +## Caches + +Both caches normally live under `~/.cache/whichllm/`. If `XDG_CACHE_HOME` is +set to an absolute path, whichllm uses `$XDG_CACHE_HOME/whichllm/` instead. + +| File | TTL | Contents | +| --- | --- | --- | +| `models.json` | 6 hours | Serialized `ModelInfo` data | +| `benchmark.json` | 24 hours | Combined benchmark score map | + +`--refresh` bypasses the relevant cache and writes a new one after fetching. + +## Benchmark sources + +`models/benchmark.py` builds one score map from multiple sources. Scores are +normalized to a 0-100 scale before ranking. + +whichllm separates sources into two tiers: + +| Tier | Sources | Treatment | +| --- | --- | --- | +| Current | LiveBench, Artificial Analysis, Aider, Vision | Overrides frozen scores for the same model | +| Frozen | Open LLM Leaderboard v2, Chatbot Arena ELO | Kept for older coverage, capped and recency-demoted | + +Current sources can use live scrapes when reachable and curated snapshots when +the upstream page shape changes. The snapshot month is printed below rankings. + +Frozen-only scores are demoted by model lineage. This prevents an older model +with a stale leaderboard score from outranking a newer generation simply because +the newer model was never added to that frozen leaderboard. + +## Benchmark evidence + +A model can receive benchmark evidence through several paths: + +| Evidence | Meaning | +| --- | --- | +| `direct` | Exact independent benchmark match | +| `variant` | Suffix-stripped or `-Instruct` variant match | +| `base_model` | Match through HuggingFace `cardData.base_model` | +| `line_interp` | Size-aware interpolation within the same model line | +| `self_reported` | Uploader-provided `evalResults` only | +| `none` | No usable evidence | + +Inheritance is rejected when the actual model size differs too much from the +reference. This catches small draft heads, MTP heads, and unrelated forks that +would otherwise borrow a larger base model's score. + +## Family grouping + +`models/grouper.py` groups related repos by: + +1. `cardData.base_model`, when available. +2. Normalized repository names. + +The normalizer removes common suffixes such as `-GGUF`, `-AWQ`, `-GPTQ`, +`-Instruct`, `-Chat`, `-FP16`, and date suffixes. It also handles versioned +model lines such as Qwen, Llama, Mistral, and DeepSeek. + +Within a family, the ranker evaluates all members and variants but keeps only +the best result for the final table. + +## Candidate variants + +For each `ModelInfo`, `engine/ranker.py` builds candidate variants: + +- Existing GGUF files are evaluated by quantization. +- Extreme low-bit GGUF variants are skipped unless explicitly requested with + `--quant`. +- Official safetensors-only repos can receive synthetic GGUF estimates for + common community conversions such as `Q4_K_M`, `Q5_K_M`, `Q6_K`, and `Q8_0`. +- Pre-quantized repos such as AWQ, GPTQ, FP8, and BF16 are not given synthetic + GGUF variants. + +Apple Silicon and CPU-only rankings are restricted to GGUF candidates for +runtime stability. Linux with NVIDIA can also rank non-GGUF AWQ/GPTQ/FP16/BF16 +repos. + +## Ranking + +For each candidate variant: + +1. Estimate memory. +2. Check whether it can run. +3. Estimate tok/s and attach speed confidence/range metadata. +4. Resolve benchmark evidence. +5. Compute a quality score. +6. Keep the best variant for the model family. + +The final sorting key stays close to the displayed quality score, with a small +direct-benchmark bonus and a CPU-only penalty. Full-GPU candidates are already +favored inside the score through the runtime-fit and speed adjustments, so the +sort key does not add a second full-GPU bonus. + +See [Scoring](scoring.md) for the score details. + +## Output + +Output is split by surface: + +- `output/ranking.py` renders hardware panels and recommendation tables. +- `output/json_output.py` renders ranking, `plan`, and `upgrade` JSON. +- `output/plan.py` renders `plan` tables. +- `output/upgrade.py` renders upgrade comparison tables. +- `output/display.py` re-exports those functions for older imports. + +Normal ranking tables show memory required, estimated generation speed, fit +type, and published date. `--details` switches to download-oriented metadata. +Speed color is based on absolute usability, while `~` marks estimates with a +range and `?` marks low-confidence, backend-sensitive estimates. diff --git a/docs/run-snippet.md b/docs/run-snippet.md new file mode 100644 index 0000000..6905733 --- /dev/null +++ b/docs/run-snippet.md @@ -0,0 +1,161 @@ +# Run and snippet + +whichllm can do more than print recommendations: + +- `whichllm run` starts an interactive chat with a selected model. +- `whichllm snippet` prints a Python script for manual use. + +Both commands use the same model loading helpers in `cli.py`. + +## `whichllm run` + +```bash +whichllm run [MODEL_NAME] +``` + +If `MODEL_NAME` is provided, whichllm searches the fetched model list for an +exact ID, suffix match, or term match. + +If `MODEL_NAME` is omitted, whichllm ranks models for the current hardware and +uses the top result. + +Examples: + +```bash +whichllm run +whichllm run "qwen 2.5 1.5b gguf" +whichllm run "phi 3 mini gguf" --cpu-only +whichllm run "llama 3 8b gguf" --quant Q5_K_M +``` + +## How `run` executes + +`run` requires `uv` in `PATH`. + +At runtime, whichllm: + +1. Loads models from cache or HuggingFace. +2. Selects a model and quantization. +3. Generates a temporary Python chat script. +4. Runs that script with `uv run --no-project`. +5. Adds the required dependencies with repeated `--with` flags. +6. Deletes the temporary script after the chat exits. + +This keeps the project environment clean. The temporary runtime dependencies +are not added to `pyproject.toml`. + +## Supported model paths + +### GGUF + +GGUF models use: + +- `llama-cpp-python` +- `huggingface-hub` + +The generated script downloads the selected GGUF file with `hf_hub_download` +and loads it through `llama_cpp.Llama`. + +GPU behavior: + +- default: `n_gpu_layers=-1` +- `--cpu-only`: `n_gpu_layers=0` + +### AWQ + +AWQ repos are inferred from the model ID and use: + +- `transformers` +- `torch` +- `accelerate` +- `autoawq` + +### GPTQ + +GPTQ repos are inferred from the model ID and use: + +- `transformers` +- `torch` +- `accelerate` +- `auto-gptq` + +### FP16, BF16, FP8, INT8, BNB 4-bit + +Other non-GGUF repos use the Transformers path. The generated script uses +`device_map="auto"` unless `--cpu-only` is set. + +## Quantization selection + +For GGUF repos, whichllm chooses a variant by this preference order unless +`--quant` is provided: + +```text +Q4_K_M, Q4_K_S, Q5_K_M, Q5_K_S, Q6_K, Q3_K_M, Q3_K_L, Q8_0, ... +``` + +This order favors a practical balance of memory and quality. Very low-bit +variants are available when explicitly requested but are not preferred by +default. + +If the requested quantization is not available, `run` warns and falls back to +the best available match. + +## Chat behavior + +GGUF scripts call: + +```python +llm.create_chat_completion(messages=messages, stream=True) +``` + +Transformers scripts use: + +```python +tokenizer.apply_chat_template(...) +model.generate(...) +``` + +The chat loop keeps the current conversation history in memory until the +process exits. Type `exit`, `quit`, or `q` to stop. + +## `whichllm snippet` + +```bash +whichllm snippet [MODEL_NAME] +``` + +`snippet` prints a short Python example instead of running it. + +Examples: + +```bash +whichllm snippet "qwen 7b" +whichllm snippet "llama 3 8b gguf" --quant Q5_K_M +``` + +If no model is provided, `snippet` picks the most-downloaded GGUF model from +the fetched model list. This is different from `run`, which auto-ranks for the +current hardware when no model name is provided. + +## Manual execution + +The snippet output includes a suggested `uv run --no-project` command with the +needed `--with` dependencies. + +Example shape: + +```bash +uv run --no-project --with llama-cpp-python --with huggingface-hub script.py +``` + +## Practical notes + +- First run can take time because dependencies and model weights need to + download. +- HuggingFace access rules still apply. Gated models may require local + HuggingFace authentication. +- `run` is a convenience path, not a full model manager. +- `snippet` is better when you want to adapt loading code into your own project. +- Generated Transformers scripts use `trust_remote_code=True`, matching common + HuggingFace local inference patterns. Review model repos before running code + from untrusted sources. diff --git a/docs/scoring.md b/docs/scoring.md new file mode 100644 index 0000000..da4b031 --- /dev/null +++ b/docs/scoring.md @@ -0,0 +1,216 @@ +# Scoring + +whichllm does not pick the largest model that fits. It ranks candidates by a +composite score that tries to answer a more practical question: + +> Of the models that can run here, which one is likely to be the best usable +> choice? + +The source of truth is `engine/ranker.py`. + +## Inputs + +Each candidate score uses: + +- model metadata from HuggingFace +- detected or simulated hardware +- estimated VRAM/RAM fit +- estimated tok/s +- quantization type +- benchmark evidence +- downloads and likes +- source organization +- model lineage and generation + +The score is capped to `0..100`. + +## Benchmark evidence + +Independent benchmark matches are not all treated equally. + +| Source | Weight | Meaning | +| --- | ---: | --- | +| `direct` | `0.62` | Exact independent benchmark match | +| `base_model` | `0.55` | Match through `cardData.base_model` | +| `variant` | `0.50` | Suffix-stripped variant match | +| `line_interp` | `0.40` | Size-aware model-line interpolation | +| `self_reported` | `0.30` | Uploader-provided HuggingFace eval only | +| `none` | `0.00` | No benchmark evidence | + +`self_reported` evidence is intentionally weak. HuggingFace model cards can +contain useful evaluation data, but it is not the same as an independent +leaderboard. + +## Size score + +Model size is used as a rough world-knowledge proxy: + +```text +size_score = 4.2 * log2(params_b) + 9 +``` + +The result is capped at `35`. + +For dense models, `params_b` is the parameter count. For MoE models, whichllm +uses total parameters for quality because all experts contribute to stored +knowledge. Active parameters are used later for speed. + +## Quantization penalty + +Lower-bit quantization can make a larger model fit, but it also reduces quality. +The score core is multiplied by `(1 - quant_penalty)`. + +Examples: + +| Quant | Penalty | +| --- | ---: | +| `Q8_0` | `0.01` | +| `Q6_K` | `0.02` | +| `Q5_K_M` | `0.03` | +| `Q4_K_M` | `0.05` | +| `Q3_K_M` | `0.08` | +| `Q2_K` | `0.25` | +| `IQ2_XXS` | `0.40` | +| `Q1_0` | `0.55` | + +Extreme low-bit variants are excluded by default when better candidates exist. +They can still be requested explicitly with `--quant`. + +## Evidence confidence + +After benchmark and size are combined, weak evidence is dampened: + +| Evidence state | Multiplier | +| --- | ---: | +| Direct benchmark | `1.00` | +| Inherited evidence | `0.78` | +| Self-reported evidence | `0.55` | +| No benchmark | `0.55` | + +For inherited benchmark evidence, the raw score is also scaled by confidence +before entering the scoring function. Line interpolation therefore receives a +double discount: once for its interpolation confidence and once for being +inherited evidence. + +## Runtime fit + +The candidate's runtime form matters: + +| Fit | Multiplier | +| --- | ---: | +| Full GPU | `1.00` | +| Partial offload | `0.42`-`0.88`, based on spill ratio | +| CPU-only | `0.50` | + +Light partial offload is penalized less than heavy offload. MoE models receive +a milder penalty when the active parameter working set can plausibly stay on +GPU while inactive experts spill to CPU RAM. + +The final family selection key does not add a separate full-GPU bonus. Runtime +fit is already reflected in the quality score through the multiplier above and +the speed adjustment below. CPU-only results receive a small extra sort penalty +when mixed with GPU-backed candidates. + +## Speed adjustment + +Speed is treated as a usability gate. It is not the main quality signal. + +Required speed depends on fit: + +| Fit | Required speed | +| --- | ---: | +| Full GPU | `8 tok/s` | +| Partial offload | `4 tok/s` | +| CPU-only | `1.5 tok/s` | + +Candidates below the required speed receive up to `-8` points. Candidates above +it receive up to `+8` points. + +After ranking, if any candidate is at least `5 tok/s`, whichllm drops candidates +below `1.5 tok/s`. This avoids recommending models that technically fit but are +not practical to use. + +The reported speed is a point estimate, not a live benchmark. Ranking also +exposes speed confidence: + +| Confidence | Range factor | Typical cases | +| --- | ---: | --- | +| `medium` | `0.60x`-`1.60x` | Normal GPU estimates, synthetic GGUF estimates, AMD shared-memory APU MoE estimates | +| `low` | `0.35x`-`2.00x` | CPU-only, partial offload, unknown bandwidth, Apple Silicon MoE | +| `high` | `0.85x`-`1.20x` | Reserved for future measured-speed data | + +Speed cells are colored by absolute usability: red is under `4 tok/s`, yellow +is `4-10 tok/s`, green is `10-30 tok/s`, and bright green is `30+ tok/s`. `~` +marks medium-confidence estimates with a range, and `?` marks low-confidence +estimates. JSON exposes the same uncertainty data as `speed_confidence`, +`speed_range_tok_per_sec`, and `speed_notes`. + +## Source trust + +The source organization contributes a small adjustment: + +- official model organizations receive a small bonus +- trusted GGUF converters can inherit that trust +- known repackagers receive a small penalty + +The adjustment is intentionally small. It should break ties, not replace +benchmark and fit signals. + +## Popularity + +Downloads and likes act as tie-breakers. Their weight is lower when benchmark +evidence is strong and higher when evidence is weak. + +Popularity has no effect for direct benchmark matches. + +## Generation lineage + +Some benchmark sources are frozen. A model released after a frozen leaderboard +cannot appear there, while older models can keep strong but stale scores. + +whichllm uses family-specific lineage maps to avoid that inversion. Newer +generations can receive a small bonus; older generations can receive a small +penalty. This is applied carefully so direct benchmark evidence still matters. + +Examples of tracked lineages include: + +- Qwen +- Llama +- DeepSeek +- Gemma +- Phi +- Mistral +- GLM +- Kimi +- Granite +- OLMo +- T5 (incl. Flan-T5, mT5, ByT5, T5Gemma) + +## Benchmark markers + +The table score can include a marker: + +| Marker | Meaning | +| --- | --- | +| none | Direct independent benchmark evidence | +| `~` | Estimated or inherited benchmark evidence | +| `!sr` | Self-reported HuggingFace eval only | +| `?` | No benchmark evidence | + +Top-pick confidence is computed from the score gap, benchmark status, and fit +type. Partial-offload and CPU-only top picks are reported with lower confidence +than full-GPU direct-benchmark winners. + +## Why a smaller model can win + +A smaller model can outrank a larger one when it has: + +- stronger current benchmark evidence +- a newer generation signal +- better quantization quality +- full-GPU fit instead of partial offload +- higher estimated speed +- a more trustworthy source + +That is intentional. whichllm ranks likely usable quality, not parameter count +alone. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..f2941be --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,423 @@ +# Troubleshooting + +This page lists common issues and the first checks to make. + +## No GPU detected + +Run: + +```bash +whichllm hardware +``` + +If an NVIDIA GPU is missing: + +- check that the driver is installed +- check `nvidia-smi` +- check that `nvidia-ml-py` can load NVML + +whichllm falls back to `nvidia-smi`, but it still needs the NVIDIA driver tools +to be working. + +If an AMD GPU is missing: + +- on Linux, check `rocm-smi`, `lspci`, and `/sys/class/drm` +- on Windows, check that PowerShell can read `Win32_VideoController` +- for Ryzen AI / Radeon integrated graphics, check whether `whichllm hardware` + shows shared memory instead of a tiny 512 MB or 4 GB adapter + +If an Intel iGPU is missing: + +- Linux detection uses `lspci` or `/sys/class/drm` +- Windows detection uses `Win32_VideoController` +- many Intel iGPUs do not expose dedicated VRAM, so they may be shown as shared + memory graphics + +## Simulate hardware instead + +If detection is unavailable or you are planning a purchase, use `--gpu`: + +```bash +whichllm --gpu "RTX 4090" +whichllm hardware --gpu "Apple M3 Max" +whichllm --gpu "RTX 5060 Ti" --vram 16 +whichllm --gpu "2x RTX 4090" +whichllm --gpu "RTX 4090" --gpu "RTX 3090" +``` + +Use `--vram` when the GPU name has multiple memory variants or is not in the +database. + +For detected iGPU or unified-memory systems, override the usable GPU memory and +bandwidth directly: + +```bash +whichllm hardware --vram 8 --ram-bandwidth 68 +whichllm --vram 8 --bandwidth 68 +``` + +If `whichllm hardware` lists multiple GPUs, add `--gpu-index` with the GPU +number from that output. + +`--vram` only applies to one simulated or detected GPU. For multi-GPU +simulation, use known GPU names and omit `--vram`. + +## `--cpu-only` conflicts with `--gpu` + +These flags are mutually exclusive: + +```bash +whichllm --cpu-only --gpu "RTX 4090" +``` + +Choose one: + +```bash +whichllm --cpu-only +whichllm --gpu "RTX 4090" +``` + +## `--vram` / `--bandwidth` needs a GPU + +These overrides need either a detected GPU or a simulated GPU: + +```bash +whichllm --vram 8 --ram-bandwidth 68 +whichllm --gpu "RTX 3060" --vram 12 +``` + +If no GPU is detected, use `--gpu` to simulate one or check the detection steps +above. + +## No compatible models found + +Try: + +```bash +whichllm +whichllm --cpu-only +whichllm --refresh +``` + +Common causes: + +- the selected `--quant` is too restrictive +- `--gpu-only` or `--fit full-gpu` filters out partial-offload and CPU-only candidates +- `--speed usable`, `--speed fast`, or `--min-speed` filters out slower candidates +- `--min-speed` is too high +- `--evidence strict` filters out all candidates +- the requested context length is too large +- available RAM is too low after reserving space for the OS +- disk free space is too low for the model weights + +For very small machines, remove optional filters first: + +```bash +whichllm --top 20 +``` + +## Recommendations use RAM or CPU offload, but I only want VRAM + +By default, whichllm includes any runnable candidate: full-GPU, partial-offload, +and CPU-only. This is useful for finding what can run at all, but it can be too +loose when you want only models that fit entirely in GPU VRAM. + +Use: + +```bash +whichllm --gpu-only +whichllm --fit gpu +whichllm --fit full-gpu +``` + +If no rows are shown, this machine has no ranked candidates that fit fully in +GPU memory under the current filters. Remove `--gpu-only`, lower the context +length, or try a smaller quantization. + +## A model fits, but it is too slow + +The default ranking table shows estimated generation speed. Slow rows are red, +marginal rows are yellow, usable rows are green, and fast rows are bright +green. The `~` and `?` markers are confidence markers for the estimate. + +Filter slow rows with: + +```bash +whichllm --speed usable # >=10 tok/s +whichllm --speed fast # >=30 tok/s +whichllm --min-speed 4 # exact floor, if you want a lower threshold +``` + +For an exact threshold: + +```bash +whichllm --min-speed 10 +``` + +## LM Studio or another runtime says the model barely does not fit + +whichllm estimates model memory, but real runtimes can need extra room for +loader overhead, graph buffers, KV cache choices, and OS pressure. By default, +whichllm reserves a small automatic VRAM headroom before fit checks. + +Tune it with: + +```bash +whichllm --vram-headroom 1.5GB +whichllm --vram-headroom 10% +whichllm --vram-headroom none +``` + +Use `none` when you want the old raw-VRAM behavior. + +## RAM offload depends on what else is running + +Partial offload uses system RAM. If Docker, Elasticsearch, a browser, or +another workload is already using a large amount of memory, cap the offload +budget: + +```bash +whichllm --ram-budget available +whichllm --ram-budget 8GB +whichllm --ram-budget 50% +``` + +`available` reads the current available RAM from the OS at startup. Fixed +values are useful when you know how much memory you want to leave for other +processes. + +## Results look stale + +whichllm caches model data for 6 hours and benchmark data for 24 hours. + +Force a refresh: + +```bash +whichllm --refresh +whichllm plan "qwen 7b" --refresh +``` + +The caches live under: + +```text +~/.cache/whichllm/ +``` + +If `XDG_CACHE_HOME` is set to an absolute path, the caches live under: + +```text +$XDG_CACHE_HOME/whichllm/ +``` + +## `uvx` fails with `realpath: command not found` + +Some older macOS versions do not include a `realpath` command. If the `uvx` +launcher fails before whichllm starts, with output like: + +```text +realpath: command not found +/Users/.../python: No such file or directory +``` + +run whichllm through Python's module entry point instead: + +```bash +uvx --from whichllm python -m whichllm +``` + +Pass normal whichllm arguments after the module name: + +```bash +uvx --from whichllm python -m whichllm --gpu "RTX 4090" +``` + +## The top pick has `~`, `!sr`, or `?` + +These markers describe benchmark evidence: + +| Marker | Meaning | +| --- | --- | +| `~` | Inherited or interpolated benchmark evidence | +| `!sr` | Uploader-reported benchmark only | +| `?` | No benchmark evidence | + +Use stricter evidence when you want only independently matched benchmark data: + +```bash +whichllm --evidence strict +whichllm --direct +``` + +Use `--evidence base` when base-model matches are acceptable but interpolation +and self-reported values are not. + +## The largest model did not win + +That is expected. whichllm scores: + +- benchmark quality +- model size +- quantization loss +- full GPU vs partial offload vs CPU-only +- estimated speed +- evidence confidence +- source trust +- generation lineage + +A smaller current-generation model with strong direct evidence can beat a +larger model that only barely fits or relies on stale benchmark data. + +## Estimated speed differs from real speed + +Speed is an estimate based on: + +- model weight size +- MoE active parameters +- GPU memory bandwidth +- quantization efficiency +- backend factor +- partial-offload penalty + +Real performance depends on the inference runtime, driver, prompt length, +batching, thermal limits, and background memory pressure. + +The default ranking table shows the speed estimate and its confidence marker. +Use `--details` only when you want download counts instead. + +Speed colors and markers: + +- red: slow generation speed, under 4 tok/s +- yellow: marginal generation speed, 4-10 tok/s +- green: usable generation speed, 10-30 tok/s +- bright green: fast local generation speed, 30+ tok/s +- `~`: estimated speed range is available +- `?`: low-confidence estimate; runtime/backend differences can be large + +JSON includes the same information as `speed_confidence`, +`speed_range_tok_per_sec`, and `speed_notes`. + +## Apple Silicon partial offload looks different + +Apple Silicon uses unified memory. Partial offload does not cross a discrete +PCIe boundary, so whichllm applies a milder speed penalty than it does for +discrete GPUs. + +The same is true for recognized AMD shared-memory APUs such as Strix Halo, +Ryzen AI MAX, and Ryzen AI / Radeon 890M-class integrated graphics. +DGX Spark / NVIDIA GB10 is handled the same way when NVIDIA reports GPU memory +as unavailable. + +On Windows, `Win32_VideoController.AdapterRAM` can cap around 4 GB. whichllm +uses the 64-bit registry memory value when it is available, and treats known +shared-memory APUs as unified-memory style devices instead of tiny discrete +GPUs. + +## `run` says `uv is required` + +Install `uv` first: + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Then retry: + +```bash +whichllm run +``` + +## `run` cannot download a model + +Possible causes: + +- the model is gated on HuggingFace +- local HuggingFace authentication is missing +- the selected GGUF filename no longer exists +- network access failed +- disk space is too low + +Try a known public GGUF model first: + +```bash +whichllm run "qwen 2.5 1.5b gguf" +``` + +## Hugging Face API access fails or needs a mirror + +whichllm uses the Hugging Face API to fetch model metadata. If direct access to +`huggingface.co` fails in your network, set `HF_ENDPOINT` to a compatible +endpoint root: + +```powershell +$env:HF_ENDPOINT = "https://huggingface.co" +whichllm --refresh +``` + +```bash +HF_ENDPOINT="https://huggingface.co" whichllm --refresh +``` + +Do not include `/api` in `HF_ENDPOINT`; whichllm adds that path internally. + +## How much disk space does `run` need? + +Normal ranking commands do not download model weights. They cache Hugging Face +model metadata and benchmark metadata under the whichllm cache. + +`whichllm run` downloads the selected GGUF file through `huggingface_hub`. The +required disk space is roughly the selected GGUF file size plus normal Hugging +Face cache overhead. + +By default, Hugging Face stores downloaded files under: + +```text +~/.cache/huggingface/hub +``` + +You can move that cache by setting `HF_HOME` or `HF_HUB_CACHE`. + +Cleanup is handled by the Hugging Face cache tools: + +```bash +hf cache scan +hf cache delete +``` + +whichllm does not currently delete model files automatically after a run. + +## Ollama names do not match HuggingFace IDs + +JSON output returns HuggingFace repo IDs: + +```bash +whichllm --top 1 --json | jq -r '.models[0].model_id' +``` + +Ollama model names often use a different naming scheme. Map the HuggingFace ID +to your local Ollama model name before calling `ollama run`. + +## Debugging a specific model + +Use `plan` to inspect memory requirements: + +```bash +whichllm plan "Qwen2.5-72B" --quant Q4_K_M +whichllm plan "Qwen2.5-72B" --quant Q8_0 --context-length 32768 +``` + +Use plain output when filing issues: + +```bash +whichllm --gpu "RTX 4090" --json +whichllm --gpu "RTX 4090" --markdown +whichllm hardware +``` + +Include: + +- OS +- GPU name and VRAM +- CPU and RAM +- command used +- expected result +- actual result diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..89eb955 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +[project] +name = "whichllm" +version = "0.5.15" +description = "Find the best LLM that runs on your hardware" +authors = [{name = "Andyyyy64"}] +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +homepage = "https://github.com/Andyyyy64/whichllm" +repository = "https://github.com/Andyyyy64/whichllm" +keywords = ["llm", "local-llm", "gpu", "vram", "huggingface", "inference", "hardware", "cli", "recommendation"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: System :: Hardware", +] +dependencies = [ + "typer>=0.9", + "rich>=13.0", + "httpx>=0.27", + "psutil>=5.9", + "dbgpu[fuzz]>=1.0", + "nvidia-ml-py>=12.0", +] + +[project.urls] +Homepage = "https://github.com/Andyyyy64/whichllm" +Repository = "https://github.com/Andyyyy64/whichllm" +"Bug Tracker" = "https://github.com/Andyyyy64/whichllm/issues" +Changelog = "https://github.com/Andyyyy64/whichllm/blob/main/CHANGELOG.md" + +[project.scripts] +whichllm = "whichllm.cli:app" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/whichllm"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "pyarrow>=23.0.1", +] diff --git a/scripts/import_livebench_csv.py b/scripts/import_livebench_csv.py new file mode 100644 index 0000000..adba1da --- /dev/null +++ b/scripts/import_livebench_csv.py @@ -0,0 +1,116 @@ +"""Convert a LiveBench leaderboard CSV into the inlined Python dict. + +LiveBench publishes their leaderboard as a dated CSV (e.g. +``https://livebench.ai/table_2026_01_08.csv``). + +Usage: + curl https://livebench.ai/table_2026_01_08.csv | python scripts/import_livebench_csv.py + +""" + +from __future__ import annotations + +import csv +import sys + +# LiveBench CSV model name -> list of HuggingFace ids that share the score. +# When several CSV rows map onto the same HF id (e.g. thinking vs. base), +# the highest average wins. +CSV_NAME_TO_HF_IDS: dict[str, list[str]] = { + "deepseek-v3.2": ["deepseek-ai/DeepSeek-V3.2"], + "deepseek-v3.2-exp": ["deepseek-ai/DeepSeek-V3.2-Exp"], + "deepseek-v3.2-exp-thinking": ["deepseek-ai/DeepSeek-V3.2-Exp"], + "deepseek-v3.2-thinking": ["deepseek-ai/DeepSeek-V3.2"], + "deepseek-v4-flash": ["deepseek-ai/DeepSeek-V4-Flash"], + "deepseek-v4-pro": ["deepseek-ai/DeepSeek-V4-Pro"], + "devstral-2512": ["mistralai/Devstral-2512"], + "gemma-4-31b-it": ["google/gemma-4-31b-it"], + "glm-4.6": ["zai-org/GLM-4.6"], + "glm-4.6v": ["zai-org/GLM-4.6V"], + "glm-4.7": ["zai-org/GLM-4.7"], + "glm-5": ["zai-org/GLM-5"], + "glm-5.1": ["zai-org/GLM-5.1"], + "gpt-oss-120b": ["openai/gpt-oss-120b"], + "kimi-k2-instruct": ["moonshotai/Kimi-K2-Instruct"], + "kimi-k2-thinking": ["moonshotai/Kimi-K2-Thinking"], + "kimi-k2.5-thinking": ["moonshotai/Kimi-K2.5"], + "kimi-k2.6-thinking": ["moonshotai/Kimi-K2.6-Thinking"], + "mimo-v2-pro": ["XiaomiMiMo/MiMo-V2-Pro"], + "minimax-m2.5": ["MiniMaxAI/MiniMax-M2.5"], + "minimax-m2.7": ["MiniMaxAI/MiniMax-M2.7"], + "nemotron-3-super-120b-a12b": ["nvidia/Nemotron-3-Super-120B-A12B"], + "qwen3-235b-a22b-instruct-2507": ["Qwen/Qwen3-235B-A22B-Instruct-2507"], + "qwen3-235b-a22b-thinking-2507": ["Qwen/Qwen3-235B-A22B-Thinking-2507"], + "qwen3-30b-a3b-thinking": ["Qwen/Qwen3-30B-A3B-Thinking-2507"], + "qwen3-32b-thinking": ["Qwen/Qwen3-32B"], + "qwen3-next-80b-a3b-instruct": ["Qwen/Qwen3-Next-80B-A3B-Instruct"], + "qwen3-next-80b-a3b-thinking": ["Qwen/Qwen3-Next-80B-A3B-Thinking"], + "qwen3.6-27b": ["Qwen/Qwen3.6-27B"], +} + + +def row_average(row: dict[str, str]) -> float | None: + nums: list[float] = [] + for key, value in row.items(): + if key == "model" or not value: + continue + try: + nums.append(float(value)) + except ValueError: + continue + if not nums: + return None + return sum(nums) / len(nums) + + +def main(argv: list[str]) -> int: + rows = list(csv.DictReader(sys.stdin)) + + best: dict[str, float] = {} + matched: set[str] = set() + for row in rows: + name = row["model"] + hf_ids = CSV_NAME_TO_HF_IDS.get(name) + if not hf_ids: + continue + avg = row_average(row) + if avg is None: + continue + matched.add(name) + for hf_id in hf_ids: + if avg > best.get(hf_id, 0.0): + best[hf_id] = avg + + unmapped_open = [ + row["model"] + for row in rows + if row["model"] not in matched + and not any( + tok in row["model"] + for tok in ( + "claude", + "gpt-", + "gemini", + "grok", + "arcee", + "elephant", + ) + ) + ] + if unmapped_open: + print( + f"# note: {len(unmapped_open)} unmapped row(s) in CSV — extend " + "CSV_NAME_TO_HF_IDS if any are open-weight:", + " ".join(sorted(unmapped_open)), + file=sys.stderr, + ) + + print("{") + for hf_id in sorted(best): + print(f' "{hf_id}": {best[hf_id]:.1f},') + print("}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/src/whichllm/__init__.py b/src/whichllm/__init__.py new file mode 100644 index 0000000..c55dc64 --- /dev/null +++ b/src/whichllm/__init__.py @@ -0,0 +1 @@ +"""whichllm: Find the best LLM that runs on your hardware.""" diff --git a/src/whichllm/__main__.py b/src/whichllm/__main__.py new file mode 100644 index 0000000..53d3c5c --- /dev/null +++ b/src/whichllm/__main__.py @@ -0,0 +1,7 @@ +"""Module entry point for ``python -m whichllm``.""" + +from whichllm.cli import app + + +if __name__ == "__main__": + app() diff --git a/src/whichllm/cli.py b/src/whichllm/cli.py new file mode 100644 index 0000000..46b1b71 --- /dev/null +++ b/src/whichllm/cli.py @@ -0,0 +1,1507 @@ +"""CLI entry point using typer.""" + +from __future__ import annotations + +import asyncio +import re +import sys +from typing import Optional + +import typer +from rich.console import Console + +from whichllm.constants import _GiB +from whichllm.hardware.types import HardwareInfo +from whichllm.models.artifacts import ( + attach_resolved_artifacts, + find_gguf_variant, + resolve_ranked_gguf_artifact, +) +from whichllm.models.types import ModelInfo +from whichllm.utils import _current_version, CONTEXT_LENGTH + +app = typer.Typer( + name="llm-checker", + help="Find the best LLM that runs on your hardware.", + no_args_is_help=False, + invoke_without_command=True, +) +console = Console() + +_find_gguf_variant = find_gguf_variant +_resolve_ranked_gguf_for_run = resolve_ranked_gguf_artifact + + +def _run_async(coro): + """Run async coroutine from sync context.""" + return asyncio.run(coro) + + +def _format_fetch_error(error: Exception) -> str: + """Return a useful one-line fetch error even when str(error) is empty.""" + detail = str(error).strip() + if detail: + return detail + + response = getattr(error, "response", None) + request = getattr(error, "request", None) or getattr(response, "request", None) + status_code = getattr(response, "status_code", None) + url = getattr(request, "url", None) + if status_code and url: + return f"{type(error).__name__}: HTTP {status_code} for {url}" + if url: + return f"{type(error).__name__} while requesting {url}" + return f"{type(error).__name__} with no detail from the network layer" + + +def _print_version(value: bool) -> None: + """Print version and exit when --version is requested.""" + if value: + console.print(_current_version()) + raise typer.Exit() + + +def _validate_gpu_flags( + cpu_only: bool, + gpu: list[str] | None, + vram: float | None, + bandwidth: float | None = None, + gpu_index: int | None = None, +) -> None: + """Validate mutual exclusivity of GPU-related flags.""" + if cpu_only and gpu: + console.print("[red]Error:[/] --cpu-only and --gpu are mutually exclusive.") + raise typer.Exit(code=1) + if cpu_only and (vram is not None or bandwidth is not None): + console.print("[red]Error:[/] --cpu-only cannot be used with GPU overrides.") + raise typer.Exit(code=1) + if vram is not None and vram <= 0: + console.print("[red]Error:[/] --vram must be greater than 0.") + raise typer.Exit(code=1) + if bandwidth is not None and bandwidth <= 0: + console.print("[red]Error:[/] --bandwidth must be greater than 0.") + raise typer.Exit(code=1) + if gpu_index is not None and gpu_index < 0: + console.print("[red]Error:[/] --gpu-index must be 0 or greater.") + raise typer.Exit(code=1) + if gpu_index is not None and gpu: + console.print("[red]Error:[/] --gpu-index only applies to detected GPUs.") + raise typer.Exit(code=1) + if gpu_index is not None and vram is None and bandwidth is None: + console.print("[red]Error:[/] --gpu-index requires --vram or --bandwidth.") + raise typer.Exit(code=1) + + +def _validate_output_flags(json_output: bool, markdown_output: bool) -> None: + """Validate mutually exclusive output formats.""" + if json_output and markdown_output: + console.print("[red]Error:[/] --json and --markdown are mutually exclusive.") + raise typer.Exit(code=1) + + +def _validate_ranking_flags( + top: int, + min_speed: float | None, + min_params: float | None, +) -> None: + """Validate ranking/filter flags that otherwise silently distort output. + + Without these guards a non-positive ``--top`` reaches ``results[:top_n]`` in + :func:`whichllm.engine.ranker.rank_models`: ``--top 0`` returns no + recommendations at all, and a negative value slices from the end + (``results[:-5]``), silently returning a truncated, unrequested subset + instead of the count the user asked for. Negative ``--min-speed`` / + ``--min-params`` thresholds are likewise meaningless. Fail fast with a clear + message instead of producing misleading results. + """ + if top < 1: + console.print("[red]Error:[/] --top must be 1 or greater.") + raise typer.Exit(code=1) + if min_speed is not None and min_speed < 0: + console.print("[red]Error:[/] --min-speed must be 0 or greater.") + raise typer.Exit(code=1) + if min_params is not None and min_params < 0: + console.print("[red]Error:[/] --min-params must be 0 or greater.") + raise typer.Exit(code=1) + + +def _validate_profile(profile: str) -> str: + """Validate ranking profile option.""" + valid = {"general", "coding", "vision", "math", "any"} + p = profile.lower() + if p not in valid: + console.print( + "[red]Error:[/] --profile must be one of: general, coding, vision, math, any." + ) + raise typer.Exit(code=1) + return p + + +def _validate_evidence(evidence: str) -> str: + """Validate evidence mode option.""" + valid = {"strict", "base", "any"} + mode = evidence.lower() + if mode not in valid: + console.print("[red]Error:[/] --evidence must be one of: strict, base, any.") + raise typer.Exit(code=1) + return mode + + +def _resolve_evidence_mode(evidence: str, direct: bool) -> str: + """Resolve final evidence mode, keeping --direct as strict alias.""" + mode = _validate_evidence(evidence) + if direct: + # 互換性維持のため --direct は strict と同義に固定する。 + return "strict" + return mode + + +def _resolve_fit_filter(fit: str, gpu_only: bool) -> str: + """Resolve runtime fit filtering, keeping --gpu-only as a short alias.""" + mode = fit.lower().replace("_", "-").replace(" ", "-") + if mode not in {"any", "gpu", "full-gpu", "fullgpu"}: + console.print("[red]Error:[/] --fit must be one of: any, gpu, full-gpu.") + raise typer.Exit(code=1) + if gpu_only: + return "full_gpu" + return "full_gpu" if mode in {"gpu", "full-gpu", "fullgpu"} else "any" + + +def _resolve_speed_filter(speed: str, min_speed: float | None) -> float | None: + """Resolve named speed presets while preserving --min-speed as exact input.""" + if min_speed is not None: + return min_speed + mode = speed.lower().replace("_", "-") + presets = { + "any": None, + "usable": 10.0, + "fast": 30.0, + } + if mode not in presets: + console.print("[red]Error:[/] --speed must be one of: any, usable, fast.") + raise typer.Exit(code=1) + return presets[mode] + + +_MEMORY_RE = re.compile( + r"^(?P\d+(?:\.\d+)?)\s*(?Pgib|gb|g|mib|mb|m)?$", + re.IGNORECASE, +) + + +def _parse_memory_amount( + value: str, *, option_name: str, total_bytes: int | None = None +) -> int: + """Parse memory CLI values. Bare numbers are treated as GiB.""" + raw = value.strip() + if not raw: + console.print(f"[red]Error:[/] {option_name} cannot be empty.") + raise typer.Exit(code=1) + + if raw.endswith("%"): + if total_bytes is None: + console.print(f"[red]Error:[/] {option_name} percentage needs a base size.") + raise typer.Exit(code=1) + try: + pct = float(raw[:-1]) + except ValueError: + console.print(f"[red]Error:[/] Invalid {option_name}: {value!r}.") + raise typer.Exit(code=1) + if pct < 0: + console.print(f"[red]Error:[/] {option_name} must be non-negative.") + raise typer.Exit(code=1) + return int(total_bytes * pct / 100.0) + + match = _MEMORY_RE.match(raw) + if not match: + console.print( + f"[red]Error:[/] Invalid {option_name}: {value!r}. " + "Use values like 1.5GB, 512MB, 10%, or 8." + ) + raise typer.Exit(code=1) + + number = float(match.group("number")) + unit = (match.group("unit") or "gb").lower() + if number < 0: + console.print(f"[red]Error:[/] {option_name} must be non-negative.") + raise typer.Exit(code=1) + + if unit in {"gib", "gb", "g"}: + return int(number * _GiB) + return int(number * 1024**2) + + +def _auto_vram_headroom(vram_bytes: int) -> int: + """Default runtime headroom so near-edge fits do not over-promise.""" + if vram_bytes <= 0: + return 0 + return int(max(512 * 1024**2, min(vram_bytes * 0.05, 2 * _GiB))) + + +def _parse_vram_headroom(value: str, vram_bytes: int) -> int: + mode = value.strip().lower() + if mode == "auto": + return _auto_vram_headroom(vram_bytes) + if mode in {"none", "off", "0"}: + return 0 + return _parse_memory_amount( + value, + option_name="--vram-headroom", + total_bytes=vram_bytes, + ) + + +def _apply_memory_budgets( + hardware: HardwareInfo, + *, + vram_headroom: str, + ram_budget: str | None, +) -> HardwareInfo: + """Apply user-facing memory budgets without mutating detected raw sizes.""" + headroom_mode = vram_headroom.strip().lower() + if not hardware.gpus and headroom_mode not in {"auto", "none", "off", "0"}: + _parse_memory_amount( + vram_headroom, + option_name="--vram-headroom", + total_bytes=_GiB, + ) + + reserved_values: list[int] = [] + for gpu in hardware.gpus: + reserved = _parse_vram_headroom(vram_headroom, gpu.vram_bytes) + gpu.usable_vram_bytes = max(0, gpu.vram_bytes - reserved) + if reserved > 0: + reserved_values.append(reserved) + + if reserved_values: + unique_reserved = sorted(set(reserved_values)) + if len(unique_reserved) == 1: + note = f"VRAM headroom: {_format_budget_bytes(unique_reserved[0])} reserved per GPU" + else: + note = "VRAM headroom: auto reserve applied per GPU" + hardware.budget_notes.append(note) + + if ram_budget: + mode = ram_budget.strip().lower() + if mode == "available": + from whichllm.hardware.memory import detect_available_ram_bytes + + hardware.ram_budget_bytes = detect_available_ram_bytes() + hardware.budget_notes.append( + f"RAM budget: current available {_format_budget_bytes(hardware.ram_budget_bytes)}" + ) + elif mode not in {"auto", "none", "off"}: + hardware.ram_budget_bytes = _parse_memory_amount( + ram_budget, option_name="--ram-budget", total_bytes=hardware.ram_bytes + ) + hardware.budget_notes.append( + f"RAM budget: {_format_budget_bytes(hardware.ram_budget_bytes)}" + ) + return hardware + + +def _format_budget_bytes(value: int) -> str: + if value >= _GiB: + return f"{value / _GiB:.1f} GB" + if value >= 1024**2: + return f"{value / 1024**2:.0f} MB" + return f"{value / 1024:.0f} KB" + + +def _apply_gpu_overrides( + hardware: HardwareInfo, + cpu_only: bool, + gpu: list[str] | None, + vram: float | None, + bandwidth: float | None = None, + gpu_index: int | None = None, +) -> HardwareInfo: + """Replace hardware.gpus based on CLI flags.""" + if cpu_only: + hardware.gpus = [] + elif gpu: + from whichllm.hardware.gpu_simulator import create_synthetic_gpus + + try: + hardware.gpus = create_synthetic_gpus(gpu, vram) + except ValueError as e: + console.print(f"[red]Error:[/] {e}") + raise typer.Exit(code=1) + if bandwidth is not None: + if len(hardware.gpus) != 1: + console.print( + "[red]Error:[/] --bandwidth currently supports exactly one " + "simulated GPU." + ) + raise typer.Exit(code=1) + hardware.gpus[0].memory_bandwidth_gbps = bandwidth + elif vram is not None or bandwidth is not None: + if not hardware.gpus: + console.print( + "[red]Error:[/] --vram/--bandwidth requires a detected GPU or --gpu." + ) + raise typer.Exit(code=1) + if gpu_index is None: + if len(hardware.gpus) > 1: + console.print( + "[red]Error:[/] --gpu-index is required when overriding " + "detected hardware with multiple GPUs." + ) + raise typer.Exit(code=1) + target_gpu = hardware.gpus[0] + else: + if gpu_index >= len(hardware.gpus): + console.print( + f"[red]Error:[/] --gpu-index {gpu_index} is out of range " + f"for {len(hardware.gpus)} detected GPU(s)." + ) + raise typer.Exit(code=1) + target_gpu = hardware.gpus[gpu_index] + + if vram is not None: + target_gpu.vram_bytes = int(vram * _GiB) + target_gpu.usable_vram_bytes = None + target_gpu.vram_overridden = True + if bandwidth is not None: + target_gpu.memory_bandwidth_gbps = bandwidth + return hardware + + +def _auto_min_params_for_profile(hardware: HardwareInfo, profile: str) -> float | None: + """Pick automatic min-params threshold for strongest general ranking. + + The threshold rises with VRAM so a 24GB GPU is steered away from 3-4B + toys, but tiny GPUs (4-8GB) still see full-GPU options instead of being + forced into 7B+ partial-offload-only results. + """ + if profile != "general": + return None + if not hardware.gpus: + return 2.0 # CPU-only: tiny is the only practical choice + from whichllm.hardware.memory import effective_usable_ram + + usable_ram = effective_usable_ram(hardware.ram_bytes, hardware.ram_budget_bytes) + best_vram_gb = max( + ( + usable_ram + if g.shared_memory + and (g.vram_bytes == 0 or hardware.ram_budget_bytes is not None) + else ( + g.usable_vram_bytes if g.usable_vram_bytes is not None else g.vram_bytes + ) + ) + for g in hardware.gpus + ) / (1024**3) + if best_vram_gb >= 30: + return 12.0 + if best_vram_gb >= 20: + return 10.0 + if best_vram_gb >= 12: + return 8.0 + if best_vram_gb >= 8: + return 5.0 + if best_vram_gb >= 5: + return 3.0 + return 2.0 + + +def _include_vision_candidates(profile: str) -> bool: + """候補取得時にVLMを含めるべきプロファイルか判定する。""" + return profile.lower() in {"vision", "any"} + + +def _fill_missing_published_at( + all_models: list, + results: list, + fetch_model_published_at, +) -> bool: + """上位表示で欠けている公開日時を補完し、更新有無を返す。""" + missing_ids = [r.model.id for r in results if not r.model.published_at] + if not missing_ids: + return False + published_map = _run_async(fetch_model_published_at(missing_ids)) + if not published_map: + return False + + updated = False + for model in all_models: + published_at = published_map.get(model.id) + if published_at and not model.published_at: + model.published_at = published_at + updated = True + return updated + + +def _merge_model_eval_benchmarks( + models: list, + benchmark_scores: dict[str, float], +) -> tuple[dict[str, float], int]: + """Deprecated no-op kept for backward API compatibility. + + Previously this injected each model's uploader-reported ``hf_eval`` + value into the leaderboard scores dict under the model's id, which + caused those values to be treated as ``direct`` benchmark evidence + by the ranker. That elevated any account that wrote a high number + in their model card to the top of the rankings. + + The hf_eval value is now consumed inside ``rank_models`` via + ``BenchmarkEvidence.source == "self_reported"`` with a much lower + weight and a dedicated display tag, so we no longer need to mutate + the leaderboard dict here. Returning the input unchanged keeps any + external callers working. + """ + return benchmark_scores, 0 + + +@app.callback(invoke_without_command=True) +def main( + ctx: typer.Context, + show_version: bool = typer.Option( + False, + "--version", + help="Show version and exit", + callback=_print_version, + is_eager=True, + ), + refresh: bool = typer.Option( + False, "--refresh", help="Ignore cache and re-fetch models" + ), + top: int = typer.Option(10, "--top", "-n", help="Number of top models to show"), + context_length: int = typer.Option( + 4096, + "--context-length", + "-c", + click_type=CONTEXT_LENGTH, + help="Context length for KV cache estimation (e.g. 4096, 64k, 128k)", + ), + quant: Optional[str] = typer.Option( + None, "--quant", "-q", help="Filter by quantization type (e.g. Q4_K_M)" + ), + min_speed: Optional[float] = typer.Option( + None, "--min-speed", help="Exact minimum tok/s filter" + ), + speed: str = typer.Option( + "any", + "--speed", + help="Speed preset filter: any | usable | fast", + ), + fit: str = typer.Option( + "any", + "--fit", + help="Runtime fit filter: any | gpu | full-gpu", + ), + gpu_only: bool = typer.Option( + False, + "--gpu-only", + help="Only show models that fit fully in GPU VRAM", + ), + evidence: str = typer.Option( + "any", + "--evidence", + help="Benchmark evidence filter: strict | base | any", + ), + direct: bool = typer.Option( + False, + "--direct", + help="Alias of --evidence strict", + ), + status: bool = typer.Option( + False, + "--status", + help="Show runtime columns (default; kept for compatibility)", + ), + details: bool = typer.Option( + False, + "--details", + help="Show Downloads metadata instead of runtime columns", + ), + min_params: Optional[float] = typer.Option( + None, + "--min-params", + help="Minimum effective parameter size in billions (e.g. 7)", + ), + profile: str = typer.Option( + "general", + "--profile", + help="Ranking profile: general | coding | vision | math | any", + ), + json_output: bool = typer.Option(False, "--json", help="Output as JSON"), + markdown_output: bool = typer.Option( + False, + "--markdown", + "-m", + help="Output as GitHub-Flavored Markdown", + ), + cpu_only: bool = typer.Option( + False, "--cpu-only", help="Ignore GPU and run in CPU-only mode" + ), + gpu: Optional[list[str]] = typer.Option( + None, + "--gpu", + help="Simulate GPU(s), e.g. 'RTX 4090', '2x RTX 4090', or repeat --gpu", + ), + vram: Optional[float] = typer.Option( + None, + "--vram", + help="Override simulated GPU VRAM or detected GPU usable VRAM in GB", + ), + bandwidth: Optional[float] = typer.Option( + None, + "--bandwidth", + "--ram-bandwidth", + help="Override GPU/RAM bandwidth in GB/s", + ), + gpu_index: Optional[int] = typer.Option( + None, + "--gpu-index", + help="Detected GPU index to override when multiple GPUs are present", + ), + vram_headroom: str = typer.Option( + "auto", + "--vram-headroom", + help="Reserve GPU memory for runtime overhead: auto | none | 1GB | 10%", + ), + ram_budget: Optional[str] = typer.Option( + None, + "--ram-budget", + help="RAM budget for offload: available | 8GB | 50%", + ), +): + """Detect hardware and recommend the best local LLMs.""" + if ctx.invoked_subcommand is not None: + return + + _validate_gpu_flags(cpu_only, gpu, vram, bandwidth, gpu_index) + _validate_output_flags(json_output, markdown_output) + _validate_ranking_flags(top, min_speed, min_params) + profile = _validate_profile(profile) + evidence_mode = _resolve_evidence_mode(evidence, direct) + fit_filter = _resolve_fit_filter(fit, gpu_only) + speed_filter = _resolve_speed_filter(speed, min_speed) + + from rich.progress import Progress, SpinnerColumn, TextColumn + + from whichllm.engine.ranker import rank_models + from whichllm.hardware.detector import detect_hardware + from whichllm.models.benchmark import ( + fetch_benchmark_scores, + load_benchmark_cache, + save_benchmark_cache, + ) + from whichllm.models.cache import load_cache, save_cache + from whichllm.models.fetcher import ( + dicts_to_models, + fetch_model_published_at, + fetch_models, + models_to_dicts, + ) + from whichllm.models.grouper import group_models + from whichllm.output.display import ( + display_hardware, + display_json, + display_markdown, + display_ranking, + ) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + # Step 1: Detect hardware + task = progress.add_task("Detecting hardware...", total=None) + hardware = detect_hardware() + _apply_gpu_overrides(hardware, cpu_only, gpu, vram, bandwidth, gpu_index) + _apply_memory_budgets( + hardware, vram_headroom=vram_headroom, ram_budget=ram_budget + ) + progress.update(task, description="Hardware detected") + + # Step 2: Fetch models + progress.update(task, description="Loading models...") + cached_data = None if refresh else load_cache() + if cached_data is not None: + models = dicts_to_models(cached_data) + progress.update(task, description=f"Loaded {len(models)} models from cache") + else: + progress.update(task, description="Fetching models from HuggingFace...") + try: + models = _run_async( + fetch_models(include_vision=_include_vision_candidates(profile)) + ) + save_cache(models_to_dicts(models)) + progress.update(task, description=f"Fetched {len(models)} models") + except Exception as e: + console.print( + f"[red]Error fetching models:[/] {_format_fetch_error(e)}" + ) + sys.exit(1) + + # Step 3: Fetch benchmark scores + progress.update(task, description="Loading benchmark data...") + bench_scores = None if refresh else load_benchmark_cache() + if bench_scores is None: + try: + progress.update(task, description="Fetching benchmark scores...") + bench_scores = _run_async(fetch_benchmark_scores()) + save_benchmark_cache(bench_scores) + except Exception as e: + console.print(f"[yellow]Warning:[/] Benchmark data unavailable: {e}") + bench_scores = {} + + # Step 4: Group and rank + progress.update(task, description="Ranking models...") + families = group_models(models) + + # Flatten all models with their family IDs set by grouper + all_models = [] + for family in families: + all_models.append(family.base_model) + all_models.extend(family.variants) + + # NOTE: We no longer merge uploader-reported hf_eval values into the + # leaderboard scores dict — the ranker now treats them as a separate + # "self_reported" evidence tier with much lower trust. See + # ranker.lookup_benchmark_evidence + _SOURCE_WEIGHTS. + + # general用途はGPUクラスに応じた自動しきい値で小さすぎるモデルを抑制する + auto_min_params = ( + _auto_min_params_for_profile(hardware, profile) + if min_params is None + else min_params + ) + + results = rank_models( + all_models, + hardware, + context_length=context_length, + top_n=top, + quant_filter=quant, + min_speed=speed_filter, + benchmark_scores=bench_scores, + task_profile=profile, + require_direct_top=True, + min_params_b=auto_min_params, + evidence_filter=evidence_mode, + fit_filter=fit_filter, + ) + + # 自動しきい値で候補ゼロなら緩和して表示を維持する + if not results and auto_min_params is not None and min_params is None: + results = rank_models( + all_models, + hardware, + context_length=context_length, + top_n=top, + quant_filter=quant, + min_speed=speed_filter, + benchmark_scores=bench_scores, + task_profile=profile, + require_direct_top=True, + min_params_b=None, + evidence_filter=evidence_mode, + fit_filter=fit_filter, + ) + + # 上位候補の公開日時が欠けている場合のみ補完して表示品質を上げる + if results: + attach_resolved_artifacts(results, all_models, quant_filter=quant) + try: + if _fill_missing_published_at( + all_models, results, fetch_model_published_at + ): + save_cache(models_to_dicts(models)) + except Exception as e: + progress.update( + task, description=f"Published date backfill skipped: {e}" + ) + + # Display results + empty_message = None + if fit_filter == "full_gpu": + empty_message = ( + "No full-GPU models found for this hardware. " + "Remove --gpu-only or use --fit any to include partial offload " + "and CPU-only candidates." + ) + if json_output: + display_json(results, hardware) + elif markdown_output: + display_markdown( + results, + hardware, + show_status=status or not details, + empty_message=empty_message, + ) + else: + console.print() + display_hardware(hardware) + console.print() + display_ranking( + results, + has_gpu=bool(hardware.gpus), + show_status=status or not details, + empty_message=empty_message, + ) + console.print() + + +@app.command() +def plan( + model_name: str = typer.Argument(..., help="Model name or HuggingFace repo ID"), + context_length: int = typer.Option( + 4096, + "--context-length", + "-c", + click_type=CONTEXT_LENGTH, + help="Context length for KV cache estimation (e.g. 4096, 64k, 128k)", + ), + quant: Optional[str] = typer.Option( + None, "--quant", "-q", help="Target quantization (default: Q4_K_M)" + ), + json_output: bool = typer.Option(False, "--json", help="Output as JSON"), + refresh: bool = typer.Option( + False, "--refresh", help="Ignore cache and re-fetch models" + ), +): + """Show what GPU you need to run a specific model.""" + from rich.progress import Progress, SpinnerColumn, TextColumn + + from whichllm.models.cache import load_cache, save_cache + from whichllm.models.fetcher import dicts_to_models, fetch_models, models_to_dicts + from whichllm.output.display import display_plan, display_plan_json + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Loading models...", total=None) + cached_data = None if refresh else load_cache() + if cached_data is not None: + models = dicts_to_models(cached_data) + else: + progress.update(task, description="Fetching models from HuggingFace...") + try: + models = _run_async(fetch_models(include_vision=True)) + save_cache(models_to_dicts(models)) + except Exception as e: + console.print( + f"[red]Error fetching models:[/] {_format_fetch_error(e)}" + ) + sys.exit(1) + + model = _search_model(models, model_name) + + target_quant = quant.upper() if quant else "Q4_K_M" + + if json_output: + display_plan_json(model, context_length, target_quant) + else: + console.print() + display_plan(model, context_length, target_quant) + console.print() + + +@app.command() +def upgrade( + target_gpus: list[str] = typer.Argument( + ..., + help="GPUs to compare against (e.g. 'RTX 4090' 'RTX 5090' 'H100')", + ), + context_length: int = typer.Option( + 8192, + "--context-length", + "-c", + click_type=CONTEXT_LENGTH, + help="Context length for ranking (e.g. 8192, 64k, 128k)", + ), + top: int = typer.Option(3, "--top", "-n", help="Best-N models to compare per GPU"), + profile: str = typer.Option("general", "--profile", help="Ranking profile"), + cpu_only: bool = typer.Option( + False, "--cpu-only", help="Compare against a CPU-only baseline" + ), + json_output: bool = typer.Option(False, "--json"), + refresh: bool = typer.Option(False, "--refresh"), +): + """Compare the current machine against potential GPU upgrades. + + For each GPU passed on the command line, simulate a system with the same + CPU/RAM but that GPU, run the ranker, and show the best-N models you'd + be able to run. Useful for answering "is upgrading from a 3090 to a 4090 + worth it?" — the table shows the quality jump and the speed jump for + each option. + """ + from rich.progress import Progress, SpinnerColumn, TextColumn + + from whichllm.engine.ranker import rank_models + from whichllm.hardware.detector import detect_hardware + from whichllm.hardware.gpu_simulator import create_synthetic_gpu + from whichllm.hardware.types import HardwareInfo + from whichllm.models.benchmark import ( + fetch_benchmark_scores, + load_benchmark_cache, + save_benchmark_cache, + ) + from whichllm.models.cache import load_cache, save_cache + from whichllm.models.fetcher import dicts_to_models, fetch_models, models_to_dicts + from whichllm.models.grouper import group_models + from whichllm.output.display import display_upgrade, display_upgrade_json + + profile = _validate_profile(profile) + _validate_ranking_flags(top, None, None) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Detecting hardware...", total=None) + current_hw = detect_hardware() + if cpu_only: + current_hw.gpus = [] + + progress.update(task, description="Loading models...") + cached_data = None if refresh else load_cache() + if cached_data is not None: + models = dicts_to_models(cached_data) + else: + progress.update(task, description="Fetching models from HuggingFace...") + try: + models = _run_async(fetch_models(include_vision=False)) + save_cache(models_to_dicts(models)) + except Exception as e: + console.print( + f"[red]Error fetching models:[/] {_format_fetch_error(e)}" + ) + raise typer.Exit(code=1) + + progress.update(task, description="Loading benchmark data...") + bench_scores = None if refresh else load_benchmark_cache() + if bench_scores is None: + try: + bench_scores = _run_async(fetch_benchmark_scores()) + save_benchmark_cache(bench_scores) + except Exception: + bench_scores = {} + + all_models: list = [] + for family in group_models(models): + all_models.append(family.base_model) + all_models.extend(family.variants) + + def _rank_for(hw: HardwareInfo): + min_p = _auto_min_params_for_profile(hw, profile) + results = rank_models( + all_models, + hw, + context_length=context_length, + top_n=top, + benchmark_scores=bench_scores, + task_profile=profile, + require_direct_top=True, + min_params_b=min_p, + ) + if not results and min_p is not None: + results = rank_models( + all_models, + hw, + context_length=context_length, + top_n=top, + benchmark_scores=bench_scores, + task_profile=profile, + require_direct_top=True, + min_params_b=None, + ) + return results + + progress.update(task, description="Ranking current hardware...") + current_results = _rank_for(current_hw) + + target_results: list[tuple[str, HardwareInfo, list]] = [] + for raw_name in target_gpus: + progress.update(task, description=f"Ranking {raw_name}...") + try: + synthetic = create_synthetic_gpu(raw_name) + except ValueError as e: + console.print(f"[yellow]Skipping {raw_name}:[/] {e}") + continue + sim_hw = HardwareInfo( + gpus=[synthetic], + cpu_name=current_hw.cpu_name, + cpu_cores=current_hw.cpu_cores, + has_avx2=current_hw.has_avx2, + has_avx512=current_hw.has_avx512, + ram_bytes=current_hw.ram_bytes, + disk_free_bytes=current_hw.disk_free_bytes, + os=current_hw.os, + ) + sim_results = _rank_for(sim_hw) + target_results.append((raw_name, sim_hw, sim_results)) + + if json_output: + display_upgrade_json(current_hw, current_results, target_results) + else: + console.print() + display_upgrade(current_hw, current_results, target_results) + console.print() + + +def _load_models(refresh: bool, include_vision: bool = True): + """Load models from cache or fetch from HuggingFace.""" + from whichllm.models.cache import load_cache, save_cache + from whichllm.models.fetcher import dicts_to_models, fetch_models, models_to_dicts + + cached_data = None if refresh else load_cache() + if cached_data is not None: + return dicts_to_models(cached_data) + try: + models = _run_async(fetch_models(include_vision=include_vision)) + save_cache(models_to_dicts(models)) + return models + except Exception as e: + console.print(f"[red]Error fetching models:[/] {_format_fetch_error(e)}") + sys.exit(1) + + +_SIZE_TOKEN_RE = re.compile(r"^(\d+(?:\.\d+)?)([bm])$", re.IGNORECASE) + + +def _parse_size_tokens( + terms: list[str], +) -> tuple[list[str], float | None]: + """Split query terms into non-size terms and an optional size in billions. + + Returns (remaining_terms, size_b) where size_b is None if no size token + was found. Only the first size token is used; subsequent size tokens are + kept as plain text terms. Handles 'b' (billions) and 'm' (millions). + """ + remaining = [] + size_b: float | None = None + for t in terms: + m = _SIZE_TOKEN_RE.match(t) + if m and size_b is None: + value = float(m.group(1)) + if value <= 0: + remaining.append(t) + continue + unit = m.group(2).lower() + size_b = value if unit == "b" else value / 1000.0 + else: + remaining.append(t) + return remaining, size_b + + +_ID_SIZE_RE = re.compile(r"(?:^|[-_/])(\d+(?:\.\d+)?)(b|m)(?:[-_.]|$)", re.IGNORECASE) + + +def _extract_id_size_b(model_id: str) -> float | None: + """Extract the size label from a model ID string, in billions. + + Scans for patterns like '7B', '1.7B', '500M' at word boundaries in the + model ID. Returns the first match converted to billions, or None. + """ + for m in _ID_SIZE_RE.finditer(model_id): + value = float(m.group(1)) + if value <= 0: + continue + unit = m.group(2).lower() + return value if unit == "b" else value / 1000.0 + return None + + +def _size_compatible(model: ModelInfo, size_b: float) -> bool: + """Check whether a model's parameter count is compatible with a query size. + + Uses a tolerance band of [0.7x, 1.5x] to accommodate rounding differences + (e.g. a 7B query matching a model with 7.6B actual parameters) while + rejecting adjacent model sizes (e.g. 7B vs 4B or 12B). + """ + if model.parameter_count <= 0: + return True + actual_b = model.parameter_count / 1e9 + ratio = actual_b / size_b + return 0.7 <= ratio <= 1.5 + + +def _search_model(models: list, model_name: str): + """Search for a model by name/ID. Returns single model or exits.""" + query_lower = model_name.lower() + terms = query_lower.split() + size_b = None + + matches = [m for m in models if m.id.lower() == query_lower] + if not matches: + matches = [m for m in models if m.id.lower().endswith("/" + query_lower)] + if not matches: + text_terms, size_b = _parse_size_tokens(terms) + matches = [ + m + for m in models + if all(t in m.id.lower() for t in text_terms) + and (size_b is None or _size_compatible(m, size_b)) + ] + + if not matches: + console.print(f"[red]No model found matching '{model_name}'.[/]") + suggestions = [m for m in models if any(t in m.id.lower() for t in terms)] + if suggestions: + suggestions.sort(key=lambda m: m.downloads, reverse=True) + console.print("\n[yellow]Did you mean:[/]") + for m in suggestions[:5]: + p = ( + f"{m.parameter_count / 1e9:.1f}B" + if m.parameter_count >= 1e9 + else f"{m.parameter_count / 1e6:.0f}M" + ) + console.print(f" • {m.id} ({p})") + raise typer.Exit(code=1) + + if size_b is not None: + + def _sort_key(m: ModelInfo) -> tuple: + id_size = _extract_id_size_b(m.id) + has_id_size = id_size is not None + id_dist = abs(id_size - size_b) if has_id_size else float("inf") + pc_dist = ( + abs(m.parameter_count / 1e9 - size_b) + if m.parameter_count > 0 + else float("inf") + ) + return ( + 0 if (has_id_size or m.parameter_count > 0) else 1, + id_dist, + pc_dist, + -m.downloads, + ) + + matches.sort(key=_sort_key) + else: + matches.sort(key=lambda m: m.downloads, reverse=True) + model = matches[0] + if len(matches) > 1: + console.print(f"[dim]Found {len(matches)} matches, using: {model.id}[/]") + return model + + +def _pick_gguf_variant(model, quant_filter: str | None = None): + """Pick the best GGUF variant for a model.""" + from whichllm.constants import QUANT_PREFERENCE_ORDER + + if not model.gguf_variants: + return None + + if quant_filter: + for v in model.gguf_variants: + if v.quant_type.upper() == quant_filter.upper(): + return v + console.print( + f"[yellow]Warning:[/] {quant_filter} not available, using best match." + ) + + # Pick by preference order + variant_map = {v.quant_type.upper(): v for v in model.gguf_variants} + for qt in QUANT_PREFERENCE_ORDER: + if qt in variant_map: + return variant_map[qt] + return model.gguf_variants[0] + + +def _resolve_model_deps(model, variant) -> tuple[list[str], str]: + """Determine pip dependencies and script type for a model. + + Returns (deps, script_type) where script_type is 'gguf' or 'transformers'. + """ + if variant: + return ["llama-cpp-python", "huggingface-hub"], "gguf" + + from whichllm.engine.quantization import infer_non_gguf_quant_type + + qt = infer_non_gguf_quant_type(model.id) + base = ["transformers", "torch", "accelerate"] + if qt == "AWQ": + return [*base, "autoawq"], "transformers" + if qt == "GPTQ": + return [*base, "auto-gptq"], "transformers" + return base, "transformers" + + +def _generate_chat_script(model, variant, context_length: int, cpu_only: bool) -> str: + """Generate a self-contained Python chat script for any model type.""" + if variant: + n_gpu = 0 if cpu_only else -1 + return f'''\ +from huggingface_hub import hf_hub_download +from llama_cpp import Llama + +print("Downloading {model.id} ({variant.quant_type})...") +model_path = hf_hub_download(repo_id="{model.id}", filename="{variant.filename}") +print("Loading model...") +llm = Llama( + model_path=model_path, + n_ctx={context_length}, + n_gpu_layers={n_gpu}, + verbose=False, +) +print("Ready! Type 'exit' to quit.\\n") +messages = [] +while True: + try: + user_input = input("> ") + except (KeyboardInterrupt, EOFError): + break + if user_input.strip().lower() in ("exit", "quit", "q"): + break + if not user_input.strip(): + continue + messages.append({{"role": "user", "content": user_input}}) + response = llm.create_chat_completion(messages=messages, stream=True) + full = "" + for chunk in response: + delta = chunk["choices"][0].get("delta", {{}}) + content = delta.get("content", "") + if content: + print(content, end="", flush=True) + full += content + print() + messages.append({{"role": "assistant", "content": full}}) +print("\\nBye!") +''' + + device_map = '"cpu"' if cpu_only else '"auto"' + dtype = "torch.float32" if cpu_only else '"auto"' + return f'''\ +import shutil +import tempfile +import torch +from threading import Thread +from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer + +model_id = "{model.id}" +offload_folder = tempfile.mkdtemp(prefix="whichllm_transformers_offload_") +try: + print(f"Loading {{model_id}}...") + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained( + model_id, + device_map={device_map}, + torch_dtype={dtype}, + trust_remote_code=True, + offload_folder=offload_folder, + ) + print("Ready! Type 'exit' to quit.\\n") + messages = [] + while True: + try: + user_input = input("> ") + except (KeyboardInterrupt, EOFError): + break + if user_input.strip().lower() in ("exit", "quit", "q"): + break + if not user_input.strip(): + continue + messages.append({{"role": "user", "content": user_input}}) + inputs = tokenizer.apply_chat_template( + messages, + return_tensors="pt", + return_dict=True, + add_generation_prompt=True, + ).to(model.device) + streamer = TextIteratorStreamer( + tokenizer, skip_prompt=True, skip_special_tokens=True + ) + thread = Thread( + target=model.generate, + kwargs=dict(**inputs, max_new_tokens=512, streamer=streamer), + ) + thread.start() + full = "" + for text in streamer: + print(text, end="", flush=True) + full += text + thread.join() + print() + messages.append({{"role": "assistant", "content": full}}) + print("\\nBye!") +finally: + try: + del model + except NameError: + pass + shutil.rmtree(offload_folder, ignore_errors=True) +''' + + +@app.command() +def run( + model_name: Optional[str] = typer.Argument( + None, help="Model to run (default: auto-pick best)" + ), + context_length: int = typer.Option( + 4096, + "--context-length", + "-c", + click_type=CONTEXT_LENGTH, + help="Context length (e.g. 4096, 64k, 128k)", + ), + quant: Optional[str] = typer.Option( + None, "--quant", "-q", help="Quantization type" + ), + refresh: bool = typer.Option(False, "--refresh", help="Ignore cache"), + cpu_only: bool = typer.Option(False, "--cpu-only", help="CPU-only mode"), +): + """Download and chat with a model. Picks the best one if none specified.""" + import os + import shutil + import subprocess + import tempfile + + if not shutil.which("uv"): + console.print("[red]uv is required.[/]") + console.print( + "Install: [bold]curl -LsSf https://astral.sh/uv/install.sh | sh[/]" + ) + raise typer.Exit(code=1) + + from rich.progress import Progress, SpinnerColumn, TextColumn + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Loading models...", total=None) + models = _load_models(refresh) + progress.remove_task(task) + + variant = None + if model_name: + model = _search_model(models, model_name) + else: + from whichllm.engine.ranker import rank_models + from whichllm.hardware.detector import detect_hardware + from whichllm.models.benchmark import load_benchmark_cache + from whichllm.models.grouper import group_models + + hardware = detect_hardware() + if cpu_only: + hardware.gpus = [] + bench_scores = load_benchmark_cache() or {} + families = group_models(models) + all_models = [] + for family in families: + all_models.append(family.base_model) + all_models.extend(family.variants) + + results = rank_models( + all_models, + hardware, + context_length=context_length, + top_n=5, + quant_filter=quant, + benchmark_scores=bench_scores, + ) + if not results: + console.print("[red]No runnable model found for your hardware.[/]") + raise typer.Exit(code=1) + skipped_gguf: list[str] = [] + model = None + for ranked in results: + if ranked.gguf_variant: + resolved = resolve_ranked_gguf_artifact( + ranked.model, + ranked.gguf_variant, + all_models, + quant_filter=quant, + ) + if resolved: + resolved_model, variant = resolved + if resolved_model.id != ranked.model.id: + console.print( + "[dim]Resolved GGUF runtime: " + f"{ranked.model.id} -> {resolved_model.id} " + f"({variant.quant_type})[/]" + ) + model = resolved_model + quant = variant.quant_type + break + skipped_gguf.append(ranked.model.id) + continue + + model = ranked.model + break + + if skipped_gguf: + skipped = ", ".join(skipped_gguf[:3]) + suffix = "..." if len(skipped_gguf) > 3 else "" + console.print( + "[yellow]Warning:[/] Skipped GGUF-ranked candidate(s) without " + f"a matching runnable GGUF repo: {skipped}{suffix}" + ) + if model is None: + console.print( + "[red]Error:[/] Top recommendations require GGUF builds, " + "but no matching GGUF repos were found." + ) + console.print( + "[dim]Try specifying a GGUF model explicitly, for example " + '`whichllm run "qwen gguf"`.[/]' + ) + raise typer.Exit(code=1) + + if variant is None: + variant = _pick_gguf_variant(model, quant) + deps, script_type = _resolve_model_deps(model, variant) + script = _generate_chat_script(model, variant, context_length, cpu_only) + + fmt = variant.quant_type if variant else script_type.upper() + console.print(f"\n[bold green]Running {model.id}[/] [dim]({fmt})[/]") + console.print(f"[dim]Setting up isolated env with: {', '.join(deps)}[/]\n") + + fd, script_path = tempfile.mkstemp(suffix=".py", prefix="whichllm_run_") + try: + with os.fdopen(fd, "w") as f: + f.write(script) + cmd = ["uv", "run", "--no-project"] + for dep in deps: + cmd.extend(["--with", dep]) + cmd.append(script_path) + result = subprocess.run(cmd) + raise typer.Exit(code=result.returncode) + finally: + os.unlink(script_path) + + +@app.command() +def snippet( + model_name: Optional[str] = typer.Argument( + None, help="Model to show snippet for (default: auto-pick best)" + ), + quant: Optional[str] = typer.Option( + None, "--quant", "-q", help="Quantization type" + ), + refresh: bool = typer.Option(False, "--refresh", help="Ignore cache"), +): + """Print a ready-to-run Python script for a model.""" + from rich.progress import Progress, SpinnerColumn, TextColumn + from rich.syntax import Syntax + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Loading models...", total=None) + models = _load_models(refresh) + progress.remove_task(task) + + if model_name: + model = _search_model(models, model_name) + else: + gguf_models = [m for m in models if m.gguf_variants] + if not gguf_models: + console.print("[red]No GGUF models found.[/]") + raise typer.Exit(code=1) + gguf_models.sort(key=lambda m: m.downloads, reverse=True) + model = gguf_models[0] + + variant = _pick_gguf_variant(model, quant) + deps, _ = _resolve_model_deps(model, variant) + + if variant: + code = f'''\ +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="{model.id}", + filename="{variant.filename}", + n_ctx=4096, + n_gpu_layers=-1, # -1 = all layers on GPU, 0 = CPU only + verbose=False, +) + +output = llm.create_chat_completion( + messages=[{{"role": "user", "content": "Hello!"}}], +) +print(output["choices"][0]["message"]["content"]) +''' + else: + code = f'''\ +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "{model.id}" +tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_id, device_map="auto", torch_dtype="auto", trust_remote_code=True, +) + +inputs = tokenizer("Hello!", return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=256) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +''' + + dep_str = " ".join(f"--with {d}" for d in deps) + console.print(f"\n[bold]{model.id}[/]") + console.print(f"[dim]# Run directly:[/] whichllm run '{model.id}'") + console.print(f"[dim]# Or manually:[/] uv run --no-project {dep_str} script.py\n") + console.print(Syntax(code, "python", theme="monokai")) + + +@app.command() +def hardware( + cpu_only: bool = typer.Option( + False, "--cpu-only", help="Ignore GPU and run in CPU-only mode" + ), + gpu: Optional[list[str]] = typer.Option( + None, + "--gpu", + help="Simulate GPU(s), e.g. 'RTX 4090', '2x RTX 4090', or repeat --gpu", + ), + vram: Optional[float] = typer.Option( + None, + "--vram", + help="Override simulated GPU VRAM or detected GPU usable VRAM in GB", + ), + bandwidth: Optional[float] = typer.Option( + None, + "--bandwidth", + "--ram-bandwidth", + help="Override GPU/RAM bandwidth in GB/s", + ), + gpu_index: Optional[int] = typer.Option( + None, + "--gpu-index", + help="Detected GPU index to override when multiple GPUs are present", + ), +): + """Show detected hardware information only.""" + _validate_gpu_flags(cpu_only, gpu, vram, bandwidth, gpu_index) + + from rich.progress import Progress, SpinnerColumn, TextColumn + + from whichllm.hardware.detector import detect_hardware + from whichllm.output.display import display_hardware + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Detecting hardware...", total=None) + hw = detect_hardware() + _apply_gpu_overrides(hw, cpu_only, gpu, vram, bandwidth, gpu_index) + progress.remove_task(task) + + console.print() + display_hardware(hw) + console.print() + + +if __name__ == "__main__": + app() diff --git a/src/whichllm/constants.py b/src/whichllm/constants.py new file mode 100644 index 0000000..a67354d --- /dev/null +++ b/src/whichllm/constants.py @@ -0,0 +1,54 @@ +"""Compatibility shim: curated registries now live under ``whichllm.data``. + +This module re-exports the same names so existing imports +(``from whichllm.constants import ...``) keep working. New code should import +from the specific ``whichllm.data.*`` submodule instead. +""" + +from whichllm.data.framework import ( + FRAMEWORK_OVERHEAD_BYTES, + MIN_COMPUTE_CAPABILITY_OLLAMA, + MIN_COMPUTE_CAPABILITY_VLLM, +) +from whichllm.data.gpu import ( + _GiB, + AMD_SHARED_MEMORY_APU_MARKERS, + CURATED_GPU_SPECS, + CuratedGPUSpec, + GPU_BANDWIDTH, + GPU_MEMORY_CLOCK_VARIANTS, + INTEL_PCI_DEVICE_NAMES, + NVIDIA_COMPUTE_CAPABILITY, + VULKAN_ONLY_GPUS, +) +from whichllm.data.lineage import ( + MODEL_GENERATION_BONUS_MAX, + MODEL_GENERATION_PENALTY_MAX, + MODEL_LINEAGE_VERSIONS, +) +from whichllm.data.quantization import ( + QUANT_BYTES_PER_WEIGHT, + QUANT_PREFERENCE_ORDER, + QUANT_QUALITY_PENALTY, +) + +__all__ = [ + "_GiB", + "AMD_SHARED_MEMORY_APU_MARKERS", + "CURATED_GPU_SPECS", + "CuratedGPUSpec", + "FRAMEWORK_OVERHEAD_BYTES", + "GPU_BANDWIDTH", + "GPU_MEMORY_CLOCK_VARIANTS", + "INTEL_PCI_DEVICE_NAMES", + "MIN_COMPUTE_CAPABILITY_OLLAMA", + "MIN_COMPUTE_CAPABILITY_VLLM", + "MODEL_GENERATION_BONUS_MAX", + "MODEL_GENERATION_PENALTY_MAX", + "MODEL_LINEAGE_VERSIONS", + "NVIDIA_COMPUTE_CAPABILITY", + "QUANT_BYTES_PER_WEIGHT", + "QUANT_PREFERENCE_ORDER", + "QUANT_QUALITY_PENALTY", + "VULKAN_ONLY_GPUS", +] diff --git a/src/whichllm/data/__init__.py b/src/whichllm/data/__init__.py new file mode 100644 index 0000000..64647dd --- /dev/null +++ b/src/whichllm/data/__init__.py @@ -0,0 +1 @@ +"""Curated reference data: GPU specs, quantization tiers, model lineage, framework limits.""" diff --git a/src/whichllm/data/framework.py b/src/whichllm/data/framework.py new file mode 100644 index 0000000..884a975 --- /dev/null +++ b/src/whichllm/data/framework.py @@ -0,0 +1,8 @@ +"""Framework overhead and minimum compute capability thresholds.""" + +# Framework overhead in bytes (~500MB) +FRAMEWORK_OVERHEAD_BYTES = 500_000_000 + +# Minimum compute capability for common frameworks +MIN_COMPUTE_CAPABILITY_OLLAMA = (5, 0) +MIN_COMPUTE_CAPABILITY_VLLM = (7, 0) diff --git a/src/whichllm/data/gpu.py b/src/whichllm/data/gpu.py new file mode 100644 index 0000000..ea87181 --- /dev/null +++ b/src/whichllm/data/gpu.py @@ -0,0 +1,299 @@ +"""GPU bandwidth, VRAM, NVIDIA compute capability, and GPU markers.""" + +from __future__ import annotations + +from typing import NamedTuple + +_GiB = 1024**3 + + +class CuratedGPUSpec(NamedTuple): + """Small curated spec for GPUs missing or ambiguous in dbgpu.""" + + name: str + vendor: str + vram_gb: float + memory_bandwidth_gbps: float + shared_memory: bool = False + + +AMD_SHARED_MEMORY_APU_MARKERS: tuple[str, ...] = ( + "STRIX HALO", + "STRXLGEN", + "RADEON 8050S", + "RADEON 8060S", + "RADEON 890M", + "RADEON 880M", + "RADEON 860M", + "RADEON 840M", + "RADEON 780M", + "RADEON 760M", + "RADEON 740M", + "RADEON 680M", + "RADEON 660M", + "RYZEN AI 9", + "RYZEN AI 7", + "RYZEN AI 5", + "RYZEN AI MAX", +) + +# GPU memory bandwidth in GB/s (theoretical peak) +# Key: substring matched against GPU name (case-insensitive) +GPU_BANDWIDTH: dict[str, float] = { + # NVIDIA Consumer - RTX 50 series + "RTX 5090": 1792.0, + "RTX 5080": 960.0, + "RTX 5070 Ti": 896.0, + "RTX 5070": 672.0, + "RTX 5060 Ti": 448.0, + "RTX 5060": 336.0, + "RTX 3050": 224.0, + # NVIDIA Consumer - RTX 40 series + "RTX 4090": 1008.0, + "RTX 4080 SUPER": 736.0, + "RTX 4080": 716.8, + "RTX 4070 Ti SUPER": 672.0, + "RTX 4070 Ti": 504.0, + "RTX 4070 SUPER": 504.0, + "RTX 4070": 504.0, + "RTX 4060 Ti": 288.0, + "RTX 4060": 272.0, + # NVIDIA Consumer - RTX 30 series + "RTX 3090 Ti": 1008.0, + "RTX 3090": 936.2, + "RTX 3080 Ti": 912.4, + "RTX 3080": 760.3, + "RTX 3070 Ti": 608.3, + "RTX 3070": 448.0, + "RTX 3060 Ti": 448.0, + "RTX 3060": 360.0, + # NVIDIA Consumer - RTX 20 series + "RTX 2080 Ti": 616.0, + "RTX 2080 SUPER": 496.0, + "RTX 2080": 448.0, + "RTX 2070 SUPER": 448.0, + "RTX 2070": 448.0, + "RTX 2060 SUPER": 448.0, + "RTX 2060": 336.0, + # NVIDIA Consumer - GTX 16 series + "GTX 1660 Ti": 288.0, + "GTX 1660 SUPER": 336.0, + "GTX 1660": 192.0, + "GTX 1650 SUPER": 192.0, + # GTX 1650 GDDR5 (8 Gbps x 128-bit). A later GDDR6 revision runs 192 GB/s; + # both share the TU117 die and PCI id 0x1F82, so they are disambiguated by + # memory clock via GPU_MEMORY_CLOCK_VARIANTS below. 128 is the conservative + # default used when the memory clock is unknown. + "GTX 1650": 128.0, + # NVIDIA Data Center + "H100": 3350.0, + "H200": 4800.0, + "DGX Spark": 273.0, + "GB10": 273.0, + "A100 80GB": 2039.0, + "A100 40GB": 1555.0, + "A100": 1555.0, + "RTX A3000 Laptop": 264.0, + "A6000": 768.0, + "A5000": 768.0, + "A4000": 448.0, + "L40S": 864.0, + "L40": 864.0, + "L4": 300.0, + "T4": 320.0, + "V100": 900.0, + "P100": 732.0, + # NVIDIA Kepler (legacy, Vulkan-only — no CUDA in modern llama.cpp). + # Values are theoretical peak memory bandwidth (GB/s) from NVIDIA + # datasheets. Kepler (compute capability 3.x) was dropped by CUDA 12 and + # current llama.cpp CUDA builds, so these cards run via the Vulkan backend + # on Linux. See VULKAN_ONLY_GPUS below. + "Quadro K6000": 288.0, + "Quadro K5200": 192.3, + "Quadro K4200": 173.0, + "Quadro K2200": 80.0, + "Quadro K620": 29.0, + "Quadro K420": 14.4, + "GTX 780": 288.4, + "GTX 770": 224.3, + "GTX 760": 192.2, + # AMD + "R9700": 640.0, + "RX 9070 XT": 640.0, + "RX 9070": 560.0, + "RX 9060 XT": 320.0, + "RX 7900 XTX": 960.0, + "RX 7900 XT": 800.0, + "RX 7800 XT": 624.0, + "RX 7700 XT": 432.0, + "RX 7600": 288.0, + "RX 6950 XT": 576.0, + "RX 6900 XT": 512.0, + "RX 6800 XT": 512.0, + "RX 6800": 512.0, + "RX 6750 XT": 432.0, + "RX 6700 XT": 384.0, + "RX 6700": 320.0, + "RX 6650 XT": 256.0, + "RX 6600 XT": 256.0, + "RX 6600": 224.0, + # AMD APUs / shared-memory graphics + "Ryzen AI MAX+ 395": 256.0, + "Ryzen AI MAX 395": 256.0, + "Radeon 890M": 120.0, + "Radeon 880M": 120.0, + "Radeon 860M": 90.0, + "Radeon 840M": 60.0, + "Radeon 780M": 90.0, + "Radeon 760M": 75.0, + "Radeon 740M": 60.0, + "Radeon 680M": 75.0, + "Radeon 660M": 55.0, + "Radeon 8060S": 256.0, + "Radeon 8050S": 256.0, + "Strix Halo": 256.0, + "STRXLGEN": 256.0, + "MI300X": 5300.0, + "MI250X": 3276.0, + "MI210": 1638.0, + # Intel discrete GPUs + "Arc Pro B70": 608.0, + "Battlemage G31": 608.0, + # Apple Silicon (unified memory bandwidth) + "M1 Ultra": 800.0, + "M1 Max": 400.0, + "M1 Pro": 200.0, + "M1": 68.25, + "M2 Ultra": 800.0, + "M2 Max": 400.0, + "M2 Pro": 200.0, + "M2": 100.0, + "M3 Ultra": 800.0, + "M3 Max": 400.0, + "M3 Pro": 150.0, + "M3": 100.0, + "M4 Ultra": 819.2, + "M4 Max": 546.0, + "M4 Pro": 273.0, + "M4": 120.0, + "M5 Max": 614.0, + "M5 Pro": 307.0, + "M5": 153.0, +} + +CURATED_GPU_SPECS: dict[str, CuratedGPUSpec] = { + "Arc Pro B70": CuratedGPUSpec( + name="Intel Arc Pro B70", + vendor="intel", + vram_gb=32.0, + memory_bandwidth_gbps=608.0, + ), + "Battlemage G31": CuratedGPUSpec( + name="Battlemage G31 [Intel Graphics]", + vendor="intel", + vram_gb=32.0, + memory_bandwidth_gbps=608.0, + ), +} + +INTEL_PCI_DEVICE_NAMES: dict[str, str] = { + "0xe223": "Battlemage G31 [Intel Graphics]", +} + +# NVIDIA GPU compute capability lookup (substring match, case-insensitive) +NVIDIA_COMPUTE_CAPABILITY: dict[str, tuple[int, int]] = { + # RTX 50 series (Blackwell) + "RTX 5090": (10, 0), + "RTX 5080": (10, 0), + "RTX 5070": (10, 0), + "RTX 5070 Ti": (10, 0), + "RTX 5060": (10, 0), + # RTX 40 series (Ada Lovelace) + "RTX 4090": (8, 9), + "RTX 4080": (8, 9), + "RTX 4070": (8, 9), + "RTX 4060": (8, 9), + # RTX 30 series (Ampere) + "RTX 3090": (8, 6), + "RTX 3080": (8, 6), + "RTX 3070": (8, 6), + "RTX 3060": (8, 6), + # RTX 20 series (Turing) + "RTX 2080": (7, 5), + "RTX 2070": (7, 5), + "RTX 2060": (7, 5), + # GTX 16 series (Turing) + "GTX 1660": (7, 5), + "GTX 1650 SUPER": (7, 5), + "GTX 1650": (7, 5), + # GTX 10 series (Pascal) + "GTX 1080": (6, 1), + "GTX 1070": (6, 1), + "GTX 1060": (6, 1), + # Data Center + "H100": (9, 0), + "H200": (9, 0), + "DGX Spark": (12, 1), + "GB10": (12, 1), + "A100": (8, 0), + "RTX A3000 Laptop": (8, 6), + "A6000": (8, 6), + "A5000": (8, 6), + "A4000": (8, 6), + "L40": (8, 9), + "L4": (8, 9), + "T4": (7, 5), + "V100": (7, 0), + "P100": (6, 0), + # Kepler series (compute capability 3.x) — legacy, Vulkan-only. + # CUDA 12 and current llama.cpp CUDA builds dropped Kepler support, so + # these cards only run through the Vulkan backend. See VULKAN_ONLY_GPUS. + "Quadro K6000": (3, 5), + "Quadro K5200": (3, 5), + "Quadro K4200": (3, 0), + "Quadro K2200": (3, 0), + "Quadro K620": (3, 0), + "Quadro K420": (3, 0), + "GTX 780": (3, 5), + "GTX 770": (3, 0), + "GTX 760": (3, 0), +} + +# GPUs sold under one marketing name in multiple memory configurations that the +# driver name and PCI device id cannot tell apart, resolved by max memory clock +# (MHz) at detection time. Each value is a list of (min_mem_clock_mhz, +# bandwidth_gbps), highest threshold first; the first threshold the detected +# clock meets wins. The base name stays in GPU_BANDWIDTH as the conservative +# default used when the memory clock is unknown. +# Threshold sits between the two memory-clock regimes (GDDR5 ~4 GHz vs GDDR6 +# ~6 GHz), well clear of either, so factory-OC boards do not straddle it. +GPU_MEMORY_CLOCK_VARIANTS: dict[str, list[tuple[float, float]]] = { + # GTX 1650 shipped as the original GDDR5 (8 Gbps x 128-bit = 128 GB/s) and a + # later GDDR6 revision (12 Gbps x 128-bit = 192 GB/s) on the same TU117 die + # and PCI id 0x1F82, so only the memory clock distinguishes them. Measured on + # a GDDR6 board (VBIOS 90.17.4D.00.1E): nvidia-smi reports clocks.max.memory + # = 6001 MHz, and Qwen3-1.7B Q4_K_M decodes at 75.4 tok/s, matching the + # 192 GB/s estimate (~78) and not 128's (~52). The GDDR5 board reports + # ~4001 MHz (NVIDIA spec, 8 Gbps GDDR5; not independently measured here); + # the 5500 MHz split is far from both. + "GTX 1650": [(5500.0, 192.0), (0.0, 128.0)], +} + +# Legacy NVIDIA GPUs with no CUDA support in modern llama.cpp builds. +# Kepler (compute capability 3.0/3.5) was removed in CUDA 12, so these cards +# can only be used through the Vulkan backend (Linux) in current llama.cpp. +# Entries are substrings matched case-insensitively against the GPU name, the +# same convention used by GPU_BANDWIDTH and NVIDIA_COMPUTE_CAPABILITY. +VULKAN_ONLY_GPUS: frozenset[str] = frozenset( + { + "Quadro K6000", + "Quadro K5200", + "Quadro K4200", + "Quadro K2200", + "Quadro K620", + "Quadro K420", + "GTX 780", + "GTX 770", + "GTX 760", + } +) diff --git a/src/whichllm/data/lineage.py b/src/whichllm/data/lineage.py new file mode 100644 index 0000000..95fb804 --- /dev/null +++ b/src/whichllm/data/lineage.py @@ -0,0 +1,161 @@ +"""Model lineage / generation half-order used to bonus or penalize family versions.""" + +# Generation lineage half-order. +# For each "family stem" we encode a monotone-increasing version map so that +# the ranker can apply a small bonus/penalty depending on whether a model +# represents the newest generation of its family. This avoids the situation +# where an older series with stale Open-LLM-Leaderboard data ranks above a +# newer release for which the leaderboard simply has no data yet. +# +# Each entry is a list of (regex_pattern, generation_index) tuples evaluated +# in order; first match wins. Patterns match against lowercased model_id. +# Higher index = newer. +MODEL_LINEAGE_VERSIONS: dict[str, list[tuple[str, int]]] = { + "qwen": [ + # ordered newest -> oldest so the bonus reflects the strongest claim + (r"qwen3\.6", 7), + (r"qwen3\.5", 6), + (r"qwen3-next", 6), + (r"qwen3-coder-next", 6), + (r"qwen3-omni", 5), + (r"qwen3", 5), + (r"qwq", 4), + (r"qwen2\.5", 3), + (r"qwen2(?!\.5)", 2), + (r"qwen1", 1), + (r"qwen-(7b|14b|72b)", 1), + ], + "llama": [ + (r"llama-?4\.5", 5), + (r"llama-?4", 4), + (r"llama-?3\.3", 3), + (r"llama-?3\.2", 3), + (r"llama-?3\.1", 3), + (r"meta-llama-?3(?!\.)", 2), + (r"llama-?2", 1), + ], + "deepseek": [ + (r"deepseek-v4", 5), + (r"deepseek-v3\.2", 4), + (r"deepseek-v3\.1", 4), + (r"deepseek-r1-0528", 4), + (r"deepseek-r1", 3), + (r"deepseek-v3-0324", 3), + (r"deepseek-v3(?!\.)", 3), + (r"deepseek-v2\.5", 2), + (r"deepseek-v2(?!\.5)", 1), + (r"deepseek-coder-v2", 2), + (r"deepseek-coder(?!-v2)", 1), + ], + "gemma": [ + # Avoid reading the "gemma" segment inside T5Gemma ids as a Gemma + # generation. T5Gemma is handled by the "t5" family instead. + (r"(? oldest. The bare "t5" + # fallback is boundary-guarded because "t5" is a collision-prone + # substring (e.g. "gpt5"); every named variant is matched before it. + (r"t5[-_]?gemma", 5), # T5Gemma (2025) — Gemma-adapted encoder-decoder + (r"flan-?t5", 4), # Flan-T5 instruction-tuned (2022) + (r"flan-?ul2", 4), # Flan-UL2 (2023) + (r"codet5p", 3), # CodeT5+ (2023); before codet5 since it's a superset + (r"ul2", 3), # UL2 (2022) + (r"code-?t5", 2), # CodeT5 (2021) + (r"long-?t5", 2), # LongT5 (2021) + (r"byt5", 2), # ByT5 byte-level (2021) + (r"mt5", 2), # mT5 multilingual (2020) + (r"t5-?v1[._]1", 2), # T5 v1.1 / LM-adapted (2020) + (r"(? (4*32 + 8) / 32 = 4.25 bits/weight = 0.53125 bytes. + # NVFP4: E2M1 element + one E4M3 (8-bit) scale per block of 16 weights + # (plus a negligible per-tensor FP32 scale) -> (4*16 + 8) / 16 = 4.5 bits + # = 0.5625 bytes, the same footprint as Q4_K_M. + "MXFP4": 0.53125, + "NVFP4": 0.5625, + # Sub-2-bit / ternary tiers (extremely lossy) + "Q1_0": 0.28, + "Q2_0": 0.28, + "TQ1_0": 0.21, + "TQ2_0": 0.28, + "IQ1_S": 0.21, + "IQ1_M": 0.22, + "IQ2_S": 0.275, + "IQ2_M": 0.30, + "IQ3_S": 0.40, + "IQ3_M": 0.42, + "IQ3_XS": 0.41, + "IQ4_NL": 0.5, +} + +# Quality penalty for each quantization type (fraction of quality lost) +# Sub-2-bit and ternary quants lose 30-60% of model quality - whichllm +# previously fell back to 5% which over-rewarded extreme quants. +QUANT_QUALITY_PENALTY: dict[str, float] = { + "F32": 0.0, + "F16": 0.0, + "BF16": 0.0, + "Q8_0": 0.01, + "Q6_K": 0.02, + "Q5_K_M": 0.03, + "Q5_K_S": 0.035, + "Q5_0": 0.035, + "Q4_K_M": 0.05, + "Q4_K_S": 0.055, + "Q4_0": 0.06, + # NVFP4's finer per-16 E4M3 scale recovers more accuracy than MXFP4's + # coarser per-32 E8M0 (power-of-two) scale, so NVFP4 sits on par with the + # mature 4-bit quantizers (Q4_K_M / AWQ at 0.05) while MXFP4 is slightly + # lossier. + "NVFP4": 0.05, + "MXFP4": 0.06, + "Q3_K_M": 0.08, + "Q3_K_S": 0.12, + "Q3_K_L": 0.075, + "Q2_K": 0.25, + "IQ4_XS": 0.05, + "IQ4_NL": 0.055, + "IQ3_XS": 0.16, + "IQ3_S": 0.17, + "IQ3_M": 0.16, + "IQ3_XXS": 0.18, + "IQ2_M": 0.30, + "IQ2_S": 0.32, + "IQ2_XXS": 0.40, + "IQ1_M": 0.50, + "IQ1_S": 0.55, + "Q2_0": 0.45, + "Q1_0": 0.55, + "TQ2_0": 0.45, + "TQ1_0": 0.55, +} + +# Preferred quantization types ordered from best to acceptable. +# Sub-3-bit and 1-bit ternary variants sit at the tail so they are only +# selected when nothing else is available or when explicitly requested. +QUANT_PREFERENCE_ORDER = [ + "Q4_K_M", + "Q4_K_S", + "NVFP4", + "MXFP4", + "Q5_K_M", + "Q5_K_S", + "Q6_K", + "Q3_K_M", + "Q3_K_L", + "Q8_0", + "IQ4_XS", + "IQ4_NL", + "Q4_0", + "Q5_0", + "Q3_K_S", + "F16", + "BF16", + "IQ3_M", + "IQ3_S", + "IQ3_XS", + "Q2_K", + "IQ3_XXS", + "IQ2_M", + "IQ2_S", + "IQ2_XXS", + "IQ1_M", + "IQ1_S", + "Q2_0", + "TQ2_0", + "Q1_0", + "TQ1_0", +] diff --git a/src/whichllm/engine/__init__.py b/src/whichllm/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/whichllm/engine/compatibility.py b/src/whichllm/engine/compatibility.py new file mode 100644 index 0000000..7765ce1 --- /dev/null +++ b/src/whichllm/engine/compatibility.py @@ -0,0 +1,261 @@ +"""Compatibility checking: can a model run on given hardware?""" + +from __future__ import annotations + +from whichllm.constants import _GiB +from whichllm.constants import MIN_COMPUTE_CAPABILITY_OLLAMA +from whichllm.constants import VULKAN_ONLY_GPUS +from whichllm.engine.quantization import estimate_weight_bytes +from whichllm.engine.types import CompatibilityResult +from whichllm.engine.vram import estimate_vram +from whichllm.hardware.memory import effective_usable_ram +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo + +_MULTI_GPU_FRAMEWORK_OVERHEAD_BYTES = int(0.3 * _GiB) +_MULTI_GPU_HOMOGENEOUS_UTILIZATION = 0.95 +_MULTI_GPU_HETEROGENEOUS_UTILIZATION = 0.90 + + +def _gpu_available_memory( + gpu: GPUInfo, usable_ram: int, *, ram_budget_active: bool = False +) -> int: + vram_bytes = ( + gpu.usable_vram_bytes if gpu.usable_vram_bytes is not None else gpu.vram_bytes + ) + if gpu.shared_memory and vram_bytes < 2 * _GiB and not gpu.vram_overridden: + return usable_ram + if gpu.shared_memory and ram_budget_active: + return min(vram_bytes, usable_ram) + return vram_bytes + + +def _uses_shared_system_pool(gpu: GPUInfo) -> bool: + return gpu.shared_memory and gpu.vram_bytes < 2 * _GiB and not gpu.vram_overridden + + +def _is_vulkan_only_gpu(gpu: GPUInfo) -> bool: + """Return True for legacy NVIDIA GPUs with no modern CUDA support. + + Kepler cards (compute capability 3.x) were dropped by CUDA 12 and current + llama.cpp CUDA builds, so they only run through the Vulkan backend. Matches + ``VULKAN_ONLY_GPUS`` entries as case-insensitive substrings of the GPU name, + the same convention used by the bandwidth/compute-capability lookups. + """ + if gpu.vendor != "nvidia": + return False + name_upper = gpu.name.upper() + return any(marker.upper() in name_upper for marker in VULKAN_ONLY_GPUS) + + +def _fit_candidate_gpus(gpus: list[GPUInfo]) -> list[GPUInfo]: + has_dedicated_gpu = any( + not _uses_shared_system_pool(gpu) and gpu.vram_bytes > 0 for gpu in gpus + ) + if not has_dedicated_gpu: + return gpus + return [gpu for gpu in gpus if not _uses_shared_system_pool(gpu)] + + +def _gpu_identity(gpu: GPUInfo) -> str: + name = gpu.name.lower().replace("(simulated)", "") + return " ".join(name.split()) + + +def _is_homogeneous_gpu_set(gpus: list[GPUInfo], available: list[int]) -> bool: + if not gpus: + return True + first = gpus[0] + first_identity = _gpu_identity(first) + first_available = available[0] + vram_tolerance = max(256 * 1024**2, int(first_available * 0.02)) + return all( + gpu.vendor == first.vendor + and _gpu_identity(gpu) == first_identity + and abs(gpu_available - first_available) <= vram_tolerance + for gpu, gpu_available in zip(gpus, available, strict=True) + ) + + +def _multi_gpu_effective_vram( + gpus: list[GPUInfo], + available: list[int], + warnings: list[str], +) -> tuple[int, bool, int | None]: + raw_total = sum(available) + if len(gpus) <= 1: + return raw_total, False, None + + if any(gpu.shared_memory or gpu.vendor == "apple" for gpu in gpus): + effective = max(available) + warnings.append( + "Multiple shared-memory GPUs are not pooled; using the largest " + "reported memory pool for fit checks" + ) + return effective, False, None + + homogeneous = _is_homogeneous_gpu_set(gpus, available) + utilization = ( + _MULTI_GPU_HOMOGENEOUS_UTILIZATION + if homogeneous + else _MULTI_GPU_HETEROGENEOUS_UTILIZATION + ) + overhead = min(raw_total, len(gpus) * _MULTI_GPU_FRAMEWORK_OVERHEAD_BYTES) + effective = int((raw_total - overhead) * utilization) + + warnings.append( + "Multi-GPU fit uses a conservative layer-split budget: " + f"{effective / _GiB:.1f} GB effective from {raw_total / _GiB:.1f} GB raw VRAM" + ) + if not homogeneous: + warnings.append( + "Heterogeneous multi-GPU setup: fit assumes uneven layer placement; " + "speed depends on backend split mode and interconnect" + ) + return effective, True, effective + + +def check_compatibility( + model: ModelInfo, + variant: GGUFVariant | None, + hardware: HardwareInfo, + context_length: int = 4096, +) -> CompatibilityResult: + """Check if a model+variant can run on the given hardware.""" + warnings: list[str] = [] + + vram_required = estimate_vram(model, variant, context_length) + + usable_ram = effective_usable_ram(hardware.ram_bytes, hardware.ram_budget_bytes) + + # Determine best GPU + best_gpu: GPUInfo | None = None + best_gpu_available = 0 + gpu_available_values: list[int] = [] + candidate_gpus = _fit_candidate_gpus(hardware.gpus) + ram_budget_active = hardware.ram_budget_bytes is not None + for gpu in candidate_gpus: + gpu_available = _gpu_available_memory( + gpu, usable_ram, ram_budget_active=ram_budget_active + ) + gpu_available_values.append(gpu_available) + if best_gpu is None or gpu_available > best_gpu_available: + best_gpu = gpu + best_gpu_available = gpu_available + + vram_available = sum(gpu_available_values) if gpu_available_values else 0 + fit_vram_available, uses_multi_gpu, multi_gpu_effective_vram = ( + _multi_gpu_effective_vram(candidate_gpus, gpu_available_values, warnings) + ) + if ( + len(candidate_gpus) > 1 + and not uses_multi_gpu + and any(gpu.shared_memory or gpu.vendor == "apple" for gpu in candidate_gpus) + ): + vram_available = fit_vram_available + offload_ram_available = ( + 0 + if best_gpu and (best_gpu.shared_memory or best_gpu.vendor == "apple") + else usable_ram + ) + + # Check compute capability for NVIDIA + if best_gpu and best_gpu.vendor == "nvidia" and best_gpu.compute_capability: + if best_gpu.compute_capability < MIN_COMPUTE_CAPABILITY_OLLAMA: + warnings.append( + f"Compute capability {best_gpu.compute_capability} is below " + f"minimum {MIN_COMPUTE_CAPABILITY_OLLAMA} for Ollama" + ) + + # Flag legacy Kepler GPUs that have no CUDA support in modern llama.cpp. + # They can still run, but only through the Vulkan backend on Linux. + if best_gpu and _is_vulkan_only_gpu(best_gpu): + warnings.append( + "Legacy Kepler GPU: no CUDA support in modern llama.cpp; " + "use the Vulkan backend (Linux) instead" + ) + + # Check ROCm for AMD. Windows AMD users can still use Vulkan/DirectML + # backends, so do not label the GPU path as unavailable there. + if ( + best_gpu + and best_gpu.vendor == "amd" + and hardware.os not in ("linux", "windows") + ): + warnings.append("ROCm requires Linux for AMD GPU inference") + + # Check Metal for Apple + if best_gpu and best_gpu.vendor == "apple" and hardware.os != "darwin": + warnings.append("Metal requires macOS for Apple Silicon inference") + + # Determine fit type + if fit_vram_available >= vram_required: + fit_type = "full_gpu" + can_run = True + offload_ratio = 0.0 + elif ( + fit_vram_available > 0 + and (fit_vram_available + offload_ram_available) >= vram_required + ): + fit_type = "partial_offload" + can_run = True + offload_ratio = ( + (vram_required - fit_vram_available) / vram_required + if vram_required > 0 + else 0.0 + ) + offload_pct = offload_ratio * 100 + if best_gpu and (best_gpu.shared_memory or best_gpu.vendor == "apple"): + warnings.append("Will use shared system memory") + else: + warnings.append( + f"~{offload_pct:.0f}% of layers will be offloaded to CPU RAM" + ) + elif usable_ram >= vram_required: + fit_type = "cpu_only" + can_run = True + offload_ratio = 0.0 + warnings.append("Will run on CPU only (much slower)") + else: + fit_type = "cpu_only" + can_run = False + offload_ratio = 0.0 + warnings.append("Insufficient memory (GPU VRAM + RAM) to run this model") + + # Context length warning + context_fits = not ( + model.context_length is not None and model.context_length < context_length + ) + if not context_fits: + warnings.append( + f"Model max context {model.context_length} < requested " + f"{context_length}; runtime will truncate or reject" + ) + elif ( + context_length > 8192 + and model.context_length + and model.context_length >= context_length + ): + warnings.append( + f"Large context ({context_length}) increases VRAM usage significantly" + ) + + # File size vs disk space + file_size = estimate_weight_bytes(model, variant) + if hardware.disk_free_bytes > 0 and file_size > hardware.disk_free_bytes: + warnings.append("Insufficient disk space to download this model") + can_run = False + + return CompatibilityResult( + model=model, + gguf_variant=variant, + can_run=can_run, + vram_required_bytes=vram_required, + vram_available_bytes=vram_available, + offload_ratio=offload_ratio, + uses_multi_gpu=uses_multi_gpu, + multi_gpu_effective_vram_bytes=multi_gpu_effective_vram, + warnings=warnings, + fit_type=fit_type, + context_fits=context_fits, + ) diff --git a/src/whichllm/engine/performance.py b/src/whichllm/engine/performance.py new file mode 100644 index 0000000..dc21a59 --- /dev/null +++ b/src/whichllm/engine/performance.py @@ -0,0 +1,284 @@ +"""Token generation speed estimation.""" + +from __future__ import annotations + +from whichllm.engine.quantization import estimate_weight_bytes +from whichllm.engine.quantization import effective_quant_type +from whichllm.hardware.types import GPUInfo +from whichllm.models.types import GGUFVariant, ModelInfo + + +# Per-quant efficiency factors applied to the theoretical bandwidth-bound +# tok/s. These reflect empirical llama.cpp / vLLM measurements: 4-bit GGUFs +# achieve the highest fraction of memory-bandwidth-limited theoretical +# throughput because the dequantization kernel is fast and weight reads +# dominate; 8-bit and FP16 drop because more compute is required per byte. +_QUANT_EFFICIENCY: dict[str, float] = { + "F32": 0.30, + "F16": 0.40, + "BF16": 0.40, + "Q8_0": 0.45, + "Q6_K": 0.50, + "Q5_K_M": 0.52, + "Q5_K_S": 0.52, + "Q5_0": 0.50, + "Q4_K_M": 0.55, + "Q4_K_S": 0.55, + "Q4_0": 0.53, + # 4-bit microscaling floats decode through native FP4 tensor-core paths + # (e.g. Blackwell) where weight reads dominate, so they land in the same + # high-efficiency band as the best 4-bit GGUF kernels. + "NVFP4": 0.56, + "MXFP4": 0.55, + "Q3_K_M": 0.50, + "Q3_K_S": 0.48, + "Q3_K_L": 0.50, + "Q2_K": 0.45, + "IQ4_XS": 0.52, + "IQ4_NL": 0.50, + "IQ3_S": 0.45, + "IQ3_M": 0.45, + "IQ3_XS": 0.45, + "IQ3_XXS": 0.42, + "IQ2_S": 0.40, + "IQ2_M": 0.40, + "IQ2_XXS": 0.38, + "IQ1_M": 0.35, + "IQ1_S": 0.35, + "Q2_0": 0.38, + "Q1_0": 0.32, + "TQ2_0": 0.35, + "TQ1_0": 0.32, +} + +_DEFAULT_QUANT_EFFICIENCY = 0.45 + +# Vendor / backend multiplier applied on top of quant efficiency. CUDA on +# modern data-center GPUs is the reference (1.0); Apple's Metal kernel is +# behind on dequantization; ROCm trails further; older CUDA generations +# also drop. +_BACKEND_FACTOR: dict[str, float] = { + "nvidia": 1.00, + "amd": 0.78, + "apple": 0.82, + "intel": 0.65, +} + +# MoE decode is partly bandwidth-bound and partly kernel/dispatch-bound. +# The old fixed 25% read floor matched high-bandwidth CUDA cards reasonably +# well, but badly under-estimated low-bandwidth unified-memory APUs such as +# Strix Halo where the active expert reads dominate. Model this as a floor +# that rises with bandwidth: ~5% at 256 GB/s, capped at the legacy 25%. +_MOE_REFERENCE_BANDWIDTH_GBPS = 256.0 +_MOE_MIN_READ_RATIO_AT_REFERENCE = 0.05 +_MOE_MAX_READ_RATIO_FLOOR = 0.25 + +_SPEED_CONFIDENCE_RANGE_FACTORS: dict[str, tuple[float, float]] = { + "high": (0.85, 1.20), + "medium": (0.60, 1.60), + "low": (0.35, 2.00), +} + +_SPEED_CONFIDENCE_ORDER = { + "low": 0, + "medium": 1, + "high": 2, +} + + +def _backend_factor(gpu: GPUInfo) -> float: + if gpu.vendor in _BACKEND_FACTOR: + return _BACKEND_FACTOR[gpu.vendor] + return 0.7 + + +def _quant_efficiency(model: ModelInfo, variant: GGUFVariant | None) -> float: + quant = effective_quant_type(model, variant) + if not quant: + return _DEFAULT_QUANT_EFFICIENCY + return _QUANT_EFFICIENCY.get(quant.upper(), _DEFAULT_QUANT_EFFICIENCY) + + +def _moe_effective_read_ratio(model: ModelInfo, gpu: GPUInfo) -> float: + """Return fraction of stored weights read per generated token for MoE.""" + if not model.is_moe or not model.parameter_count_active: + return 1.0 + if model.parameter_count <= 0: + return 1.0 + + active_ratio = model.parameter_count_active / model.parameter_count + if active_ratio <= 0: + return 1.0 + + bandwidth = gpu.memory_bandwidth_gbps or 0.0 + if bandwidth > 0: + floor = _MOE_MIN_READ_RATIO_AT_REFERENCE * max( + 1.0, bandwidth / _MOE_REFERENCE_BANDWIDTH_GBPS + ) + else: + floor = _MOE_MAX_READ_RATIO_FLOOR + floor = min(_MOE_MAX_READ_RATIO_FLOOR, floor) + + return min(1.0, max(active_ratio, floor)) + + +def _lower_speed_confidence(current: str, candidate: str) -> str: + if _SPEED_CONFIDENCE_ORDER[candidate] < _SPEED_CONFIDENCE_ORDER[current]: + return candidate + return current + + +def _looks_synthetic_gguf(model: ModelInfo, variant: GGUFVariant | None) -> bool: + if variant is None: + return False + if not variant.filename: + return False + expected = f"{model.name}.{variant.quant_type}.gguf" + return variant.filename == expected + + +def estimate_speed_uncertainty( + model: ModelInfo, + variant: GGUFVariant | None, + gpu: GPUInfo | None, + fit_type: str, + estimated_tok_per_sec: float | None, +) -> tuple[str, tuple[float, float] | None, list[str]]: + """Return confidence metadata for the speed point estimate. + + The tok/s estimator is intentionally hardware/model-metadata based; it + does not know the user's exact llama.cpp, Vulkan, ROCm, Metal, MLX, or + runtime kernel versions. This helper keeps that uncertainty visible + without mixing it into the ranking score itself. + """ + notes = [ + "Speed is estimated from memory bandwidth, quantization, backend, and fit type." + ] + confidence = "medium" + + if estimated_tok_per_sec is None or estimated_tok_per_sec <= 0: + return ( + "low", + None, + notes + ["No usable bandwidth estimate was available for this setup."], + ) + + if gpu is None or fit_type == "cpu_only": + confidence = "low" + notes.append( + "CPU-only speed varies heavily with memory channels and BLAS/kernel path." + ) + else: + if not gpu.memory_bandwidth_gbps: + confidence = "low" + notes.append( + "GPU memory bandwidth is unknown, so speed is especially uncertain." + ) + + if fit_type == "partial_offload": + confidence = "low" + if gpu.vendor == "apple" or gpu.shared_memory: + notes.append( + "Partial offload on unified memory is backend-sensitive but avoids a PCIe cliff." + ) + else: + notes.append( + "Partial offload on a discrete GPU depends strongly on PCIe and CPU RAM bandwidth." + ) + + if model.is_moe: + notes.append( + "MoE speed uses active parameters plus a bandwidth-scaled dispatch/read floor." + ) + if gpu.vendor == "apple": + confidence = _lower_speed_confidence(confidence, "low") + notes.append( + "Apple Silicon MoE throughput is especially sensitive to Metal/MLX runtime kernels." + ) + elif gpu.vendor == "amd" and gpu.shared_memory: + confidence = _lower_speed_confidence(confidence, "medium") + notes.append( + "AMD shared-memory APU estimates are calibrated by bandwidth, but ROCm/Vulkan kernels can differ." + ) + + if _looks_synthetic_gguf(model, variant): + confidence = _lower_speed_confidence(confidence, "medium") + notes.append( + "This is a synthetic GGUF estimate for an official repo, not a measured GGUF file." + ) + + low_factor, high_factor = _SPEED_CONFIDENCE_RANGE_FACTORS[confidence] + speed_range = ( + round(estimated_tok_per_sec * low_factor, 1), + round(estimated_tok_per_sec * high_factor, 1), + ) + return confidence, speed_range, notes + + +def estimate_tok_per_sec( + model: ModelInfo, + variant: GGUFVariant | None, + gpu: GPUInfo | None, + fit_type: str = "full_gpu", +) -> float: + """Estimate tokens per second for inference. + + Model: throughput is bounded by the time it takes to read all weights + needed per token, multiplied by quant- and backend-specific efficiency + factors. The default 0.5 efficiency factor used earlier mixed two + distinct losses (compute kernel quality and offload overhead) into one + constant — this version separates them so a Q4_K_M model on CUDA scores + differently from the same model running on Metal or with partial + offload. + """ + if gpu is None or fit_type == "cpu_only": + params_b = model.parameter_count / 1e9 + if model.is_moe and model.parameter_count_active: + params_b = model.parameter_count_active / 1e9 + if params_b <= 0: + return 0.0 + # Modern desktop CPUs sustain roughly 4-8 GB/s effective for the + # bandwidth-bound dequant+matmul loop on a single socket. Quantized + # 4-bit 7B → ~3.5 GB → ~1-2 tok/s. Approximate with an inverse-size + # heuristic that gets the right order of magnitude. + quant_factor = _quant_efficiency(model, variant) / _DEFAULT_QUANT_EFFICIENCY + return max(0.3, 18.0 / max(params_b, 0.5) * quant_factor) + + model_size = estimate_weight_bytes(model, variant) + + # MoE: use a speed-specific effective read ratio. VRAM fit still uses + # total stored weights elsewhere; this only estimates per-token reads. + if model.is_moe and model.parameter_count_active: + effective_read = model_size * _moe_effective_read_ratio(model, gpu) + else: + effective_read = model_size + + bandwidth = gpu.memory_bandwidth_gbps * 1e9 if gpu.memory_bandwidth_gbps else 0 + if bandwidth == 0: + return 0.0 + + theoretical = bandwidth / effective_read + + # Real-world efficiency depends on quant kernel and backend. + efficiency = _quant_efficiency(model, variant) * _backend_factor(gpu) + + # Partial offload penalty depends on the memory architecture: + # + # - Discrete GPU (NVIDIA/AMD/Intel): spilled weights live in CPU RAM + # and are read across PCIe at ~1/10th of VRAM bandwidth. With ~40% + # of the model offloaded the blended throughput lands near 0.45x. + # - Apple Silicon: GPU and CPU share one physical unified-memory pool. + # AMD shared-memory APUs such as Strix Halo have the same no-PCIe-cliff + # shape for model weights, even though their backend factor remains AMD. + # "Exceeding VRAM" only means exceeding the recommended working set; + # the bytes are still read from the same high-bandwidth unified RAM, + # so there is no PCIe cliff — only mild OS/cache contention. Using + # the discrete 0.45x here was the bug that made DeepSeek-R1-class + # models on M2/M3 Ultra report ~1.7 t/s when real-world is 4-15. + if fit_type == "partial_offload": + if gpu.vendor == "apple" or gpu.shared_memory: + efficiency *= 0.85 + else: + efficiency *= 0.45 + + return theoretical * efficiency diff --git a/src/whichllm/engine/quantization.py b/src/whichllm/engine/quantization.py new file mode 100644 index 0000000..e0743e8 --- /dev/null +++ b/src/whichllm/engine/quantization.py @@ -0,0 +1,82 @@ +"""Quantization helpers shared across ranking and estimators.""" + +from __future__ import annotations + +import re + +from whichllm.constants import QUANT_QUALITY_PENALTY +from whichllm.models.types import GGUFVariant, ModelInfo + +# GGUFでないリポジトリ名から量子化方式を推定する +_NON_GGUF_PATTERNS: list[tuple[str, str]] = [ + (r"(^|[-_/])awq($|[-_/])", "AWQ"), + (r"(^|[-_/])gptq($|[-_/])", "GPTQ"), + # 4-bit microscaling float formats. Anchored so they only match a distinct + # repo-name token, never a substring of an unrelated id. + (r"(^|[-_/])mxfp4($|[-_/])", "MXFP4"), + (r"(^|[-_/])nvfp4($|[-_/])", "NVFP4"), + (r"(bnb[-_/]?4bit|nf4|int4|4bit)", "BNB_4BIT"), + (r"(int8|8bit)", "INT8"), + (r"(^|[-_/])fp8($|[-_/])", "FP8"), + (r"(^|[-_/])bf16($|[-_/])", "BF16"), + (r"(^|[-_/])(fp16|f16)($|[-_/])", "FP16"), +] + +# GGUF以外の簡易推定: 重み1つあたりのバイト数 +_NON_GGUF_BYTES_PER_WEIGHT: dict[str, float] = { + "AWQ": 0.5, + "GPTQ": 0.5, + "BNB_4BIT": 0.5, + "MXFP4": 0.53125, + "NVFP4": 0.5625, + "INT8": 1.0, + "FP8": 1.0, + "BF16": 2.0, + "FP16": 2.0, +} + +# GGUF以外の簡易推定: 品質低下率 +_NON_GGUF_QUALITY_PENALTY: dict[str, float] = { + "AWQ": 0.05, + "GPTQ": 0.05, + "BNB_4BIT": 0.07, + "MXFP4": 0.06, + "NVFP4": 0.05, + "INT8": 0.02, + "FP8": 0.02, + "BF16": 0.0, + "FP16": 0.0, +} + + +def infer_non_gguf_quant_type(model_id: str) -> str: + """Infer non-GGUF quantization type from a model repo ID.""" + lower = model_id.lower() + for pattern, quant_type in _NON_GGUF_PATTERNS: + if re.search(pattern, lower): + return quant_type + return "FP16" + + +def effective_quant_type(model: ModelInfo, variant: GGUFVariant | None) -> str: + """Return effective quantization type for a model+variant pair.""" + if variant: + return variant.quant_type.upper() + return infer_non_gguf_quant_type(model.id) + + +def estimate_weight_bytes(model: ModelInfo, variant: GGUFVariant | None) -> int: + """Estimate model weight size in bytes.""" + if variant: + return variant.file_size_bytes + quant_type = infer_non_gguf_quant_type(model.id) + bytes_per_weight = _NON_GGUF_BYTES_PER_WEIGHT.get(quant_type, 2.0) + return int(model.parameter_count * bytes_per_weight) + + +def quant_quality_penalty(model: ModelInfo, variant: GGUFVariant | None) -> float: + """Return quality penalty fraction for a quantization format.""" + quant_type = effective_quant_type(model, variant).upper() + if quant_type in QUANT_QUALITY_PENALTY: + return QUANT_QUALITY_PENALTY[quant_type] + return _NON_GGUF_QUALITY_PENALTY.get(quant_type, 0.05) diff --git a/src/whichllm/engine/ranker.py b/src/whichllm/engine/ranker.py new file mode 100644 index 0000000..ccc96c7 --- /dev/null +++ b/src/whichllm/engine/ranker.py @@ -0,0 +1,885 @@ +"""Model ranking: score and select the best models for the user's hardware.""" + +from __future__ import annotations + +import math +import re + +from whichllm.constants import ( + MODEL_GENERATION_BONUS_MAX, + MODEL_GENERATION_PENALTY_MAX, + MODEL_LINEAGE_VERSIONS, + QUANT_BYTES_PER_WEIGHT, + QUANT_PREFERENCE_ORDER, +) +from whichllm.engine.compatibility import check_compatibility +from whichllm.engine.performance import estimate_speed_uncertainty, estimate_tok_per_sec +from whichllm.engine.quantization import effective_quant_type, quant_quality_penalty +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import HardwareInfo +from whichllm.models.benchmark import ( + BenchmarkEvidence, + build_line_bucket_index, + build_score_index, + lookup_benchmark_evidence, +) +from whichllm.models.types import GGUFVariant, ModelInfo + +# Pre-compile lineage regex tables once at import time. +_LINEAGE_REGEX: dict[str, list[tuple[re.Pattern[str], int]]] = { + family: [(re.compile(pat), idx) for pat, idx in entries] + for family, entries in MODEL_LINEAGE_VERSIONS.items() +} +_LINEAGE_FAMILY_MAX: dict[str, int] = { + family: max(idx for _, idx in entries) for family, entries in _LINEAGE_REGEX.items() +} +_MULTI_GPU_SPEED_FACTOR = 0.70 + + +def _family_selection_key( + result: CompatibilityResult, + require_direct_top: bool, +) -> tuple[float]: + """Family-level selection key — single composite score. + + ``quality_score`` already includes the runtime fit penalty and speed + adjustment. Keep final selection close to that displayed score so strong + partial-offload candidates do not get discounted again while sorting. + + - ``direct_bonus`` (+5) gives independent leaderboard evidence a + small edge at the same fit; cannot overturn a 6+ point quality gap + """ + if require_direct_top and result.benchmark_status == "direct": + direct_bonus = 5.0 + else: + direct_bonus = 0.0 + cpu_penalty = -6.0 if result.fit_type == "cpu_only" else 0.0 + ctx_penalty = -20.0 if not result.context_fits else 0.0 + return (result.quality_score + direct_bonus + cpu_penalty + ctx_penalty,) + + +def _partial_offload_quality_factor(model: ModelInfo, offload_ratio: float) -> float: + """Discount partial-offload candidates by how much leaves VRAM.""" + ratio = max(0.0, min(1.0, offload_ratio)) + if ratio >= 0.75: + factor = 0.42 + elif ratio >= 0.60: + factor = 0.52 + elif ratio >= 0.40: + factor = 0.62 + elif ratio >= 0.25: + factor = 0.76 + else: + factor = 0.86 + + # MoE offload is more nuanced: inactive experts and router/runtime + # placement do not hurt equally. If the GPU can plausibly hold the + # active expert working set, do not treat inactive-expert spill like + # dense-layer spill. + if model.is_moe and model.parameter_count_active: + active_ratio = ( + model.parameter_count_active / model.parameter_count + if model.parameter_count > 0 + else 1.0 + ) + active_ratio = max(0.0, min(1.0, active_ratio)) + active_set_fits = ratio <= max(0.0, 1.0 - active_ratio) + if active_set_fits: + if ratio >= 0.75: + factor = max(factor, 0.66) + elif ratio >= 0.60: + factor = max(factor, 0.70) + elif ratio >= 0.40: + factor = max(factor, 0.76) + elif ratio >= 0.25: + factor = max(factor, 0.82) + else: + factor = max(factor, 0.88) + else: + factor = min(0.76, factor + 0.08) + + return factor + + +# Per-source benchmark weight applied to the raw 0-100 score before it is +# combined with size, quant penalty, etc. The widest gap is between "direct" +# (independent leaderboard) and "self_reported" (uploader card claim). +_SOURCE_WEIGHTS: dict[str, float] = { + "direct": 0.62, + "base_model": 0.55, + "variant": 0.50, + "line_interp": 0.40, + "self_reported": 0.30, + "none": 0.0, +} + + +_SYNTHETIC_QUANTS = ("Q3_K_M", "Q4_K_M", "Q5_K_M", "Q6_K", "Q8_0") +_PREQUANTIZED_REPO_RE = re.compile( + r"-(awq|gptq|bnb|fp8|fp16|bf16|mxfp4|nvfp4|int4|int8|4bit|8bit|gguf)$", + re.IGNORECASE, +) + + +def _synthesize_variants_for_official_repo( + model: ModelInfo, quant_filter_upper: str | None +) -> list[GGUFVariant]: + """Return synthetic GGUF variants for popular safetensors-only repos. + + HuggingFace doesn't always index GGUF siblings for an official model + (e.g. ``Qwen/Qwen3.6-27B`` ships only safetensors), but bartowski / + lmstudio-community / QuantFactory invariably publish Q4_K_M and Q8_0 + conversions within a day of release. Without synthetic variants, we'd + score these models at BF16 file sizes (~2x larger than realistic), which + forces a partial_offload penalty on otherwise-runnable mid-size models. + + Skips repos that already advertise a specific quantization in their name + (``...-AWQ``, ``...-GPTQ``, ``...-FP8`` etc.) — those are non-GGUF formats + and synthesizing a Q4_K_M alternative would misrepresent what the repo + actually contains. + """ + org = model.id.split("/", 1)[0] if "/" in model.id else "" + if org not in _OFFICIAL_ORGS: + return [] + if _PREQUANTIZED_REPO_RE.search(model.id): + return [] + out: list[GGUFVariant] = [] + for quant in _SYNTHETIC_QUANTS: + if quant_filter_upper and quant != quant_filter_upper: + continue + bpw = QUANT_BYTES_PER_WEIGHT.get(quant, 0.5625) + out.append( + GGUFVariant( + filename=f"{model.name}.{quant}.gguf", + quant_type=quant, + file_size_bytes=int(model.parameter_count * bpw), + ) + ) + return out + + +def _iter_candidate_variants( + model: ModelInfo, + quant_filter: str | None = None, +) -> list[GGUFVariant | None]: + """Build candidate variants to evaluate for a model.""" + quant_filter_upper = quant_filter.upper() if quant_filter else None + + if not model.gguf_variants: + synthetic = _synthesize_variants_for_official_repo(model, quant_filter_upper) + if synthetic: + return list(synthetic) + quant_type = effective_quant_type(model, None) + if quant_filter_upper and quant_type != quant_filter_upper: + return [] + return [None] + + # Filter by quant type if specified + candidates: list[GGUFVariant] = model.gguf_variants + if quant_filter_upper: + candidates = [ + v for v in candidates if v.quant_type.upper() == quant_filter_upper + ] + if not candidates: + return [] + else: + # Sub-3-bit GGUFs lose 25-60% of model quality and rarely produce + # a meaningfully better candidate than a smaller model at Q4_K_M. + # Exclude them unless explicitly requested via --quant. + _EXTREME_QUANTS = { + "Q2_K", + "Q2_0", + "Q1_0", + "TQ2_0", + "TQ1_0", + "IQ3_XXS", + "IQ2_XXS", + "IQ2_S", + "IQ2_M", + "IQ1_M", + "IQ1_S", + } + filtered = [ + v for v in candidates if v.quant_type.upper() not in _EXTREME_QUANTS + ] + if filtered: + candidates = filtered + + # Sort by preference order + def variant_sort_key(v: GGUFVariant) -> int: + try: + return QUANT_PREFERENCE_ORDER.index(v.quant_type.upper()) + except ValueError: + return len(QUANT_PREFERENCE_ORDER) + + candidates = sorted(candidates, key=variant_sort_key) + + return list(candidates) + + +_OFFICIAL_ORGS = frozenset( + { + "Qwen", + "meta-llama", + "google", + "mistralai", + "deepseek-ai", + "microsoft", + "nvidia", + "01-ai", + "tiiuae", + "apple", + "CohereForAI", + "bigcode", + # 2025+ frontier open-weights labs that publish safetensors-only + # repos which the community immediately converts to GGUF. + "openai", + "zai-org", + "moonshotai", + "MiniMaxAI", + "XiaomiMiMo", + "allenai", + "ibm-granite", + "stepfun-ai", + } +) + +# Trusted GGUF converters — format converters that don't change model quality +_TRUSTED_CONVERTERS = frozenset( + { + "bartowski", + "lmstudio-community", + "QuantFactory", + "unsloth", + "ggml-org", + "Mungert", + } +) + +# Known repackagers — typically reupload others' models without added value +_REPACKAGER_ORGS = frozenset( + { + "MaziyarPanahi", + "TheBloke", + "SanctumAI", + "solidrust", + "mradermacher", + } +) + +# Orgs whose repositories ship CI fixtures, deprecated research artifacts, or +# debug binaries that are not viable production LLMs. Exclude them outright so +# they cannot occupy ranking slots regardless of download counts. +_EXCLUDED_ORGS = frozenset( + { + "openai-community", # gpt2 family, 2019 research + "distilbert", # distilgpt2 etc. + "facebook", # opt-125m research scaffolds + "EleutherAI", # pythia/gpt-neo research + "trl-internal-testing", # TRL CI fixtures + "hmellor", # random tiny test models + "HuggingFaceH4", # often staging / fixtures + "transformersbook", + "togethercomputer", # mostly inference endpoints, no GGUFs + } +) + +# Substring patterns in *names* that strongly suggest non-production usage. +_EXCLUDED_NAME_PATTERNS = ( + "tiny-", + "-tiny", + "tiny_", + "_tiny", + "test-only", + "debug-", + "playground", + "-fixture", + "for-testing", + "tiny-random", + "ci-", +) + +# Naming patterns that indicate a fine-tune / merge / "uncensoring" derivative +# of a real base model. These derivatives inherit the base model's benchmark +# score via line_interp, but the derivative itself is rarely benchmarked +# independently and frequently degrades quality. Apply a soft score penalty +# rather than full exclusion so they can still surface when nothing better is +# available. +_DUBIOUS_DERIVATIVE_PATTERNS = ( + "heretic", + "abliterat", + "uncensored", + "obliterat", + "abliter", + "horror", + "erotic", + "nsfw", + "rp-", + "-rp", + "roleplay", + "darkidol", + "darkforest", + "tiefigh", + "smaug", + "personalityengine", + "lexi", + "violence", + "violet", + "schizo", + "dark-", + "twilight", + "celeste", + "midnight-rose", + "moistral", + "stheno", + "fimbulvetr", + "wizard-vicuna", + "kunoichi", +) + + +def _derivative_name_penalty(model_id: str) -> float: + """Return a score penalty (in raw quality points) for fine-tune / + "uncensored" / merge derivatives that ride on a real base model's + benchmark line. The penalty is gentle (≤ 12pt) so a derivative can + still win when its size class has no better option. + """ + if not model_id: + return 0.0 + lower = model_id.lower() + name = lower.split("/", 1)[1] if "/" in lower else lower + for pat in _DUBIOUS_DERIVATIVE_PATTERNS: + if pat in name: + return -10.0 + return 0.0 + + +def _is_excluded_model(model_id: str) -> bool: + """Return True for CI/research/fixture models that should never rank.""" + if not model_id: + return True + org = model_id.split("/", 1)[0] if "/" in model_id else "" + if org in _EXCLUDED_ORGS: + return True + lower = model_id.lower() + name = lower.split("/", 1)[1] if "/" in lower else lower + for pat in _EXCLUDED_NAME_PATTERNS: + if pat in name: + return True + return False + + +def _generation_bonus(model_id: str) -> float: + """Return a small additive bonus reflecting how new a model's + generation is within its family. The newest version of each + recognized family gets +MODEL_GENERATION_BONUS_MAX. Older + versions get a smaller bonus (or a small penalty for the + legacy generation). Unknown families return 0. + + This is purely an additive correction to the quality score + and is small enough that strong benchmark evidence will still + dominate. + """ + if not model_id: + return 0.0 + lower = model_id.lower() + best_bonus = 0.0 + for family, patterns in _LINEAGE_REGEX.items(): + for regex, idx in patterns: + if regex.search(lower): + top = _LINEAGE_FAMILY_MAX[family] + if top <= 1: + contribution = 0.0 + else: + # Map oldest -> -PENALTY_MAX, newest -> +BONUS_MAX. + norm = (idx - 1) / (top - 1) # 0 .. 1 + span = MODEL_GENERATION_BONUS_MAX + MODEL_GENERATION_PENALTY_MAX + contribution = norm * span - MODEL_GENERATION_PENALTY_MAX + if abs(contribution) > abs(best_bonus): + best_bonus = contribution + break # first match wins for this family + return best_bonus + + +def _detect_specializations(model_id: str) -> set[str]: + """モデルIDから用途特化タグを検出する。""" + lower = model_id.lower() + tags: set[str] = set() + if re.search(r"(coder|codegen|starcoder|program|coding)", lower): + tags.add("coding") + if re.search(r"(^|[-_/])(vl|vision|multimodal|llava|image)([-_/]|$)", lower): + tags.add("vision") + if re.search(r"(^|[-_/])math([-_/]|$)", lower): + tags.add("math") + return tags + + +def _matches_profile(model: ModelInfo, task_profile: str) -> bool: + """指定プロファイルにモデルが合致するか判定する。""" + profile = task_profile.lower() + tags = _detect_specializations(model.id) + if profile == "any": + return True + if profile == "general": + return len(tags) == 0 + return profile in tags + + +def _effective_params_b(model: ModelInfo) -> float: + """Return effective parameter size in billions.""" + if model.is_moe and model.parameter_count_active: + return model.parameter_count_active / 1e9 + return model.parameter_count / 1e9 + + +def _knowledge_capacity_b(model: ModelInfo) -> float: + """Return the knowledge capacity in billions for size filtering. + + For dense models this is the parameter count. For MoE models, total + parameters (all expert weights live in VRAM and contribute to the + knowledge encoded in the model) is the right yardstick — ``min_params`` + is asking "how much does this model know?" not "how much does it + compute per token". Using *active* params here was the bug that hid + Qwen3-Next-80B-A3B from the H100 ranking — its 3B active was below the + 12B auto-floor for 30GB+ GPUs even though its 80B total clearly fits. + """ + return model.parameter_count / 1e9 + + +def _passes_evidence_filter(source: str, evidence_filter: str) -> bool: + """判定根拠フィルタに合致するかを返す。""" + mode = evidence_filter.lower() + if mode == "strict": + return source == "direct" + if mode == "base": + return source in {"direct", "variant", "base_model"} + return True + + +def _is_gguf_only_backend(hardware: HardwareInfo) -> bool: + """実行基盤の都合でGGUFのみを許可すべきか判定する。""" + # Apple Silicon(macOS/Metal)とCPU-onlyは、実運用の安定性を優先してGGUFに限定する。 + if hardware.os == "darwin": + return True + if not hardware.gpus: + return True + + # Linux + NVIDIA (CUDA) は AWQ/GPTQ 含む非GGUFも許可する。 + has_linux_nvidia = hardware.os == "linux" and any( + g.vendor == "nvidia" for g in hardware.gpus + ) + return not has_linux_nvidia + + +def _compute_quality_score( + model: ModelInfo, + variant: GGUFVariant | None, + tok_per_sec: float, + fit_type: str, + offload_ratio: float = 0.0, + family_downloads: int = 0, + family_likes: int = 0, + benchmark_avg: float | None = None, + benchmark_source: str = "none", +) -> float: + """Compute a quality score (0-100) for ranking. + + Factors: + - Benchmark score weighted by source tier + - Model size (log scale) + - Quantization penalty + - Fit type penalty (partial offload / CPU-only heavily penalized) + - Speed bonus / penalty (practical usability) + - Popularity (downloads/likes) as soft tie-breaker + - Official org bonus (vs known repackagers) + - Generation-lineage bonus (newest family member > legacy generation) + """ + params_b = model.parameter_count / 1e9 + if model.is_moe and model.parameter_count_active: + effective_b = model.parameter_count_active / 1e9 + else: + effective_b = params_b + + if effective_b <= 0: + return 0.0 + + # Benchmarks lead, but raw model size also matters: a 70B at Q4_K_M + # carries far more world knowledge than a 7B Q4_K_M even when the + # leaderboard score gap is modest. For MoE models, knowledge capacity + # tracks *total* params (every expert contributes to what the model + # knows), while routing keeps per-token compute small. Use total params + # for the size score and let the speed term separately reward MoE + # efficiency. + size_basis_b = params_b + size_score = 4.2 * math.log2(max(size_basis_b, 0.5)) + 9 + size_score = min(size_score, 35) + + has_benchmark = benchmark_avg is not None and benchmark_avg > 0 + is_direct = benchmark_source == "direct" + is_self_reported = benchmark_source == "self_reported" + is_inherited = benchmark_source in {"variant", "base_model", "line_interp"} + + bench_weight = _SOURCE_WEIGHTS.get(benchmark_source, 0.0) + benchmark_score = 0.0 + if has_benchmark: + raw = min(100.0, benchmark_avg) + benchmark_score = raw * bench_weight + + # Quantization penalty + quant_penalty = quant_quality_penalty(model, variant) + quality_core = (benchmark_score + size_score) * (1 - quant_penalty) + + # Weak / unverifiable evidence gets an extra discount. + if not has_benchmark: + quality_core *= 0.55 + elif is_self_reported: + quality_core *= 0.55 # uploader claim, easily fabricated + elif is_inherited: + quality_core *= 0.78 + + # Runtime form factor penalty + if fit_type == "partial_offload": + quality_core *= _partial_offload_quality_factor(model, offload_ratio) + elif fit_type == "cpu_only": + quality_core *= 0.50 + + # Speed acts as a usability gate rather than a ranking primary. + required_speed = ( + 8.0 + if fit_type == "full_gpu" + else (4.0 if fit_type == "partial_offload" else 1.5) + ) + if tok_per_sec > 0: + if tok_per_sec < required_speed: + speed_score = -8.0 * (1 - (tok_per_sec / required_speed)) + else: + speed_score = min(8.0, math.log2(tok_per_sec / required_speed + 1.0) * 3.2) + else: + if fit_type == "partial_offload": + if offload_ratio >= 0.70: + speed_score = -24.0 + elif offload_ratio >= 0.40: + speed_score = -18.0 + else: + speed_score = -12.0 + else: + speed_score = -8.0 + + # Popularity is a tie-breaker, never primary. + downloads = max(model.downloads, family_downloads) + likes = max(model.likes, family_likes) + pop_score_raw = 0.0 + if downloads > 0: + pop_score_raw += min(1.0, math.log10(max(downloads, 1)) / 6 * 1.0) + if likes > 0: + pop_score_raw += min(1.0, math.log10(max(likes, 1)) / 4 * 1.0) + + if is_direct: + pop_weight = 0.0 + elif is_self_reported: + pop_weight = 0.4 # uploader claim is weak — popularity acts as sanity check + elif has_benchmark: + pop_weight = 0.2 + else: + pop_weight = 0.6 + pop_score = pop_score_raw * pop_weight + + # Source-trust bonus stays small. + source_bonus_raw = 0.0 + org = model.id.split("/")[0] if "/" in model.id else "" + if org in _OFFICIAL_ORGS: + source_bonus_raw = 5.0 + elif org in _REPACKAGER_ORGS: + source_bonus_raw = -5.0 + elif model.base_model: + base_org = model.base_model.split("/")[0] if "/" in model.base_model else "" + if base_org in _OFFICIAL_ORGS: + if org in _TRUSTED_CONVERTERS: + source_bonus_raw = 5.0 + else: + source_bonus_raw = 0.0 + + if is_direct: + source_weight = 0.2 + elif is_self_reported: + source_weight = 0.5 + elif has_benchmark: + source_weight = 0.4 + else: + source_weight = 0.6 + source_bonus = source_bonus_raw * source_weight + + # Generation lineage bonus: newest in a known family gets a small boost, + # confirmed legacy versions get a small penalty. Helps surface Qwen3.6, + # DeepSeek V4, Gemma 4, etc. against accumulated download leaders. + gen_bonus = _generation_bonus(model.id) + # When benchmark evidence is missing or self-reported, the lineage signal + # carries more weight (we have less else to go on). + if not has_benchmark or is_self_reported: + gen_bonus *= 1.5 + elif is_direct: + gen_bonus *= 0.6 + + # Penalty for "uncensored / abliterated / heretic / RP" derivatives that + # ride on a base model's score without independent benchmarking. + derivative_penalty = _derivative_name_penalty(model.id) + + return max( + 0.0, + min( + 100.0, + quality_core + + speed_score + + pop_score + + source_bonus + + gen_bonus + + derivative_penalty, + ), + ) + + +def rank_models( + models: list[ModelInfo], + hardware: HardwareInfo, + context_length: int = 4096, + top_n: int = 10, + quant_filter: str | None = None, + min_speed: float | None = None, + benchmark_scores: dict[str, float] | None = None, + task_profile: str = "general", + require_direct_top: bool = True, + min_params_b: float | None = None, + evidence_filter: str = "any", + fit_filter: str = "any", +) -> list[CompatibilityResult]: + """Rank models by quality for the given hardware. Returns top N results.""" + results: list[CompatibilityResult] = [] + gguf_only_backend = _is_gguf_only_backend(hardware) + + # Pre-compute max downloads/likes per family so GGUF converters + # inherit popularity from the official base model + family_max_downloads: dict[str, int] = {} + family_max_likes: dict[str, int] = {} + # Track the parameter count of the family's dominant member (highest + # downloads). Used to detect quasi-fork uploads whose params differ + # drastically from the family proper (e.g. a 6.6B MTP-head extracted + # from a 158B base ending up tagged with the same family_id). + family_dominant_params: dict[str, int] = {} + family_dominant_downloads: dict[str, int] = {} + for m in models: + fid = m.family_id + family_max_downloads[fid] = max(family_max_downloads.get(fid, 0), m.downloads) + family_max_likes[fid] = max(family_max_likes.get(fid, 0), m.likes) + if m.parameter_count and m.downloads >= family_dominant_downloads.get(fid, -1): + family_dominant_downloads[fid] = m.downloads + family_dominant_params[fid] = m.parameter_count + + # Deduplicate by family: pick best variant per family + seen_families: set[str] = set() + + # Sort models by downloads (popular first) to process best candidates first + sorted_models = sorted(models, key=lambda m: m.downloads, reverse=True) + + # Build benchmark indices once (case-insensitive + model line) + if benchmark_scores: + bench_ci_index, bench_line_index = build_score_index(benchmark_scores) + bench_line_buckets = build_line_bucket_index(benchmark_scores) + else: + bench_ci_index, bench_line_index = {}, {} + bench_line_buckets = {} + + best_gpu = None + for gpu in hardware.gpus: + if best_gpu is None or gpu.vram_bytes > best_gpu.vram_bytes: + best_gpu = gpu + + for model in sorted_models: + if _is_excluded_model(model.id): + continue + if not _matches_profile(model, task_profile): + continue + if min_params_b is not None and _knowledge_capacity_b(model) < min_params_b: + continue + + candidates = _iter_candidate_variants(model, quant_filter) + if not candidates: + continue + + fid = model.family_id + # Uploader-reported evalResults are only ever last-resort evidence. + self_reported = None + if isinstance(model.benchmark_scores, dict): + v = model.benchmark_scores.get("hf_eval") + if isinstance(v, (int, float)) and v > 0: + self_reported = float(v) + + bench_evidence = BenchmarkEvidence(score=None, confidence=0.0, source="none") + if benchmark_scores or self_reported is not None: + actual_params_b = ( + (model.parameter_count or 0) / 1e9 if model.parameter_count else None + ) + bench_evidence = lookup_benchmark_evidence( + model.id, + model.base_model, + benchmark_scores or {}, + ci_index=bench_ci_index, + line_index=bench_line_index, + line_bucket_index=bench_line_buckets, + self_reported_score=self_reported, + actual_params_b=actual_params_b, + ) + # Family-size sanity check: if this model inherited benchmarks + # via family/base_model lookup but its own params disagree + # sharply with the family's dominant member, reject the + # inheritance. Catches MTP heads / draft / abliterated forks + # that share a family_id with their base but are effectively + # different models. + if bench_evidence.source in ("variant", "base_model", "line_interp"): + dom_params = family_dominant_params.get(model.family_id) + if dom_params and model.parameter_count and dom_params > 0: + ratio = model.parameter_count / dom_params + if ratio < 0.5 or ratio > 2.0: + bench_evidence = BenchmarkEvidence( + score=None, confidence=0.0, source="none" + ) + if not _passes_evidence_filter(bench_evidence.source, evidence_filter): + continue + + # 各variantを評価し、そのモデルで最もスコアが高いものを採用する + best_for_model: CompatibilityResult | None = None + for variant in candidates: + if gguf_only_backend and variant is None: + continue + compat = check_compatibility(model, variant, hardware, context_length) + if not compat.can_run: + continue + if fit_filter == "full_gpu" and compat.fit_type != "full_gpu": + continue + + tok_per_sec = estimate_tok_per_sec( + model, variant, best_gpu, compat.fit_type + ) + if compat.uses_multi_gpu: + tok_per_sec *= _MULTI_GPU_SPEED_FACTOR + if min_speed is not None and tok_per_sec < min_speed: + continue + + bench_avg = None + if bench_evidence.score is not None: + if bench_evidence.source in {"direct", "self_reported"}: + bench_avg = bench_evidence.score + else: + # Inherited evidence: scale by confidence so weak inheritance + # (e.g. line_interp at conf 0.22) gets discounted on top of + # the per-source weight in _compute_quality_score. + confidence = max(0.0, min(1.0, bench_evidence.confidence)) + bench_avg = bench_evidence.score * (0.75 + 0.25 * confidence) + + compat.estimated_tok_per_sec = tok_per_sec + ( + compat.speed_confidence, + compat.speed_range_tok_per_sec, + compat.speed_notes, + ) = estimate_speed_uncertainty( + model, + variant, + best_gpu, + compat.fit_type, + tok_per_sec, + ) + if compat.uses_multi_gpu: + compat.speed_confidence = "low" + if tok_per_sec > 0: + compat.speed_range_tok_per_sec = ( + round(tok_per_sec * 0.35, 1), + round(tok_per_sec * 2.0, 1), + ) + compat.speed_notes.append( + "Multi-GPU speed depends on layer/tensor split mode, " + "PCIe/NVLink bandwidth, and backend support; this estimate " + "does not assume ideal scaling." + ) + compat.quality_score = _compute_quality_score( + model, + variant, + tok_per_sec, + compat.fit_type, + offload_ratio=compat.offload_ratio, + family_downloads=family_max_downloads.get(fid, 0), + family_likes=family_max_likes.get(fid, 0), + benchmark_avg=bench_avg, + benchmark_source=bench_evidence.source, + ) + # Map evidence source to a 4-value display status. "self_reported" + # is shown distinctly so users can spot uploader-claimed numbers. + if bench_evidence.score is None: + compat.benchmark_status = "none" + elif bench_evidence.source == "direct": + compat.benchmark_status = "direct" + elif bench_evidence.source == "self_reported": + compat.benchmark_status = "self_reported" + else: + compat.benchmark_status = "estimated" + compat.benchmark_source = bench_evidence.source + compat.benchmark_confidence = bench_evidence.confidence + + if ( + best_for_model is None + or compat.quality_score > best_for_model.quality_score + ): + best_for_model = compat + + if best_for_model is None: + continue + + # Deduplicate by family: keep the one with highest quality score + family_key = model.family_id + if family_key in seen_families: + # Check if this is better than existing + existing = next( + (r for r in results if r.model.family_id == family_key), None + ) + if existing and _family_selection_key( + best_for_model, + require_direct_top, + ) > _family_selection_key(existing, require_direct_top): + results.remove(existing) + results.append(best_for_model) + continue + + seen_families.add(family_key) + results.append(best_for_model) + + if require_direct_top: + results.sort( + key=lambda r: _family_selection_key(r, require_direct_top), + reverse=True, + ) + else: + results.sort( + key=lambda r: _family_selection_key(r, require_direct_top), reverse=True + ) + + # Junk floor: when at least one candidate scores ≥ 30, drop anything + # below 20. This stops Q1_0 / Q2_0 derivatives (and other extreme-quant + # repos) from occupying ranking slots when a *real* option exists. If + # every candidate is junk (very tiny GPU + no fitting Q4) we keep the + # whole list so the user still sees what they can run. + if any(r.quality_score >= 30 for r in results): + results = [r for r in results if r.quality_score >= 20] + + # Speed floor: a model that scores well on quality but runs at <1.5 t/s + # in practice (e.g. DeepSeek-V4-Flash 158B partial-offloading 100GB to + # CPU RAM from a 4GB GTX 1650) is not actually usable. Drop these + # candidates unless every remaining option is sub-1.5 too, in which + # case the user has hardware that cannot run anything responsively + # and we still want to show what's available. + if any(r.estimated_tok_per_sec >= 5.0 for r in results): + results = [r for r in results if r.estimated_tok_per_sec >= 1.5] + + # Clamp top_n: a negative value would slice from the end + # (``results[:-5]``) and silently return a truncated, unrequested subset, + # while 0 legitimately yields an empty list. The CLI rejects non-positive + # --top, but guard here too so direct callers of this public helper never + # get a truncated ranking from a stray negative count. + return results[: max(top_n, 0)] diff --git a/src/whichllm/engine/types.py b/src/whichllm/engine/types.py new file mode 100644 index 0000000..60457e3 --- /dev/null +++ b/src/whichllm/engine/types.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from whichllm.models.types import GGUFVariant, ModelInfo + + +@dataclass +class CompatibilityResult: + model: ModelInfo + gguf_variant: GGUFVariant | None + can_run: bool + vram_required_bytes: int + vram_available_bytes: int + offload_ratio: float = 0.0 # 0.0-1.0 fraction of weights spilled to CPU RAM + estimated_tok_per_sec: float | None = None + speed_confidence: str = "medium" # "high" | "medium" | "low" + speed_range_tok_per_sec: tuple[float, float] | None = None + speed_notes: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + quality_score: float = 0.0 # 0-100 for ranking + fit_type: str = "full_gpu" # "full_gpu" | "partial_offload" | "cpu_only" + benchmark_status: str = "none" # "direct" | "estimated" | "self_reported" | "none" + benchmark_source: str = "none" # granular: "direct" | "variant" | "base_model" | "line_interp" | "self_reported" | "none" + benchmark_confidence: float = 0.0 # 0.0-1.0 from BenchmarkEvidence + context_fits: bool = True # False when known model max context < requested + uses_multi_gpu: bool = False + multi_gpu_effective_vram_bytes: int | None = None + artifact_model: ModelInfo | None = None + artifact_variant: GGUFVariant | None = None diff --git a/src/whichllm/engine/vram.py b/src/whichllm/engine/vram.py new file mode 100644 index 0000000..d5427be --- /dev/null +++ b/src/whichllm/engine/vram.py @@ -0,0 +1,100 @@ +"""VRAM usage estimation.""" + +from __future__ import annotations + +from whichllm.constants import FRAMEWORK_OVERHEAD_BYTES +from whichllm.engine.quantization import estimate_weight_bytes +from whichllm.models.types import GGUFVariant, ModelInfo + +# Empirical KV-cache coefficient: bytes per B-active-param per K-context-token +# for FP16 K/V tensors. Calibrated against published llama.cpp memory reports +# for Qwen2.5-7B (0.45 GB @ 8K), Qwen3-32B (3.1 GB @ 32K), and Llama-3.1-70B +# (5.4 GB @ 32K with GQA), then bumped slightly because real llama.cpp also +# allocates a graph-compute buffer proportional to KV size. +_KV_BYTES_PER_BPARAM_PER_KCTX = 3.5 * 1024 * 1024 # 3.5 MB + +# MoE attention scales with the *attention-layer count*, which is roughly +# proportional to active_params * this multiplier. For Qwen3-Next-80B-A3B +# (3B active, 48 layers), the multiplier lands near 4. +_MOE_ATTENTION_PARAM_MULTIPLIER = 4.0 + + +def _effective_kv_context(model: ModelInfo, context_length: int) -> float: + """Context length that actually contributes to the KV cache. + + For sliding-window-attention (SWA) models, local-attention layers only keep + the last ``sliding_window`` tokens, so their KV footprint plateaus once the + request exceeds the window. Hybrid models interleave a fraction of global + (full-context) layers; that fraction is ``sliding_window_global_ratio``. + + Effective context blends the two layer types:: + + global_ratio * ctx + (1 - global_ratio) * min(ctx, window) + + This is only applied for architectures whose mainline runtimes honor SWA + (the fetcher leaves ``sliding_window`` ``None`` otherwise), and it can only + ever lower the estimate — so a model that does not advertise an honored + window keeps the full-context KV figure and stays conservative. + """ + window = model.sliding_window + ratio = model.sliding_window_global_ratio + if not window or window <= 0 or ratio is None: + return float(context_length) + ratio = min(max(ratio, 0.0), 1.0) + windowed = min(context_length, window) + return ratio * context_length + (1.0 - ratio) * windowed + + +def estimate_kv_cache(model: ModelInfo, context_length: int) -> int: + """Estimate KV cache size in bytes for a given context length. + + Dense models: KV ≈ 3 MB × params_b × ctx_k (FP16 K+V across all layers). + MoE models: scale from active params × an empirical multiplier because + attention shares across experts. + Sliding-window models cap local-layer KV at the window size (see + :func:`_effective_kv_context`). + """ + if model.is_moe and model.parameter_count_active: + # Active-params × MoE multiplier gives a reasonable proxy for the + # attention-layer footprint without needing config.num_hidden_layers. + active_b = model.parameter_count_active / 1e9 + params_b = active_b * _MOE_ATTENTION_PARAM_MULTIPLIER + else: + params_b = model.parameter_count / 1e9 + + effective_ctx = _effective_kv_context(model, context_length) + ctx_k = effective_ctx / 1024 + kv_bytes = int(params_b * ctx_k * _KV_BYTES_PER_BPARAM_PER_KCTX) + return max(kv_bytes, 0) + + +def _activation_bytes(model: ModelInfo, context_length: int) -> int: + """Activation/scratch buffer size. + + Empirically activation memory grows mildly with both model size and + context length. The prior constant-plus-linear-param formula + over-counted small models and under-counted long contexts. + """ + # Use effective (active for MoE) size as the param-dependent base + if model.is_moe and model.parameter_count_active: + effective_p = model.parameter_count_active + else: + effective_p = model.parameter_count + + base = 400_000_000 # 400 MB framework activation floor + param_term = int(effective_p * 0.08) # ~0.08 byte/param + ctx_term = int((context_length / 4096) * 150_000_000) # +150 MB per 4K + return base + param_term + ctx_term + + +def estimate_vram( + model: ModelInfo, + variant: GGUFVariant | None, + context_length: int = 4096, +) -> int: + """Estimate total VRAM required to run a model.""" + weights = estimate_weight_bytes(model, variant) + kv_cache = estimate_kv_cache(model, context_length) + activation = _activation_bytes(model, context_length) + framework = FRAMEWORK_OVERHEAD_BYTES + return weights + kv_cache + activation + framework diff --git a/src/whichllm/hardware/__init__.py b/src/whichllm/hardware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/whichllm/hardware/amd.py b/src/whichllm/hardware/amd.py new file mode 100644 index 0000000..5122920 --- /dev/null +++ b/src/whichllm/hardware/amd.py @@ -0,0 +1,286 @@ +"""AMD GPU detection via rocm-smi with Linux fallback probes.""" + +from __future__ import annotations + +import json +import logging +import shlex +import subprocess +from pathlib import Path + +from whichllm.constants import AMD_SHARED_MEMORY_APU_MARKERS, _GiB +from whichllm.hardware.gpu_db import _static_bandwidth, resolve_detected_bandwidth +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + +_DISPLAY_CLASSES = ( + "vga compatible controller", + "3d controller", + "display controller", +) + + +def _lookup_bandwidth(name: str) -> float | None: + """Curated GPU_BANDWIDTH lookup, compound-lspci aware. Kept for regression + tests; live detection goes through ``resolve_detected_bandwidth``, which + also consults dbgpu.""" + return _static_bandwidth(name) + + +def _is_shared_memory_apu(name: str) -> bool: + name_upper = name.upper() + return any(marker in name_upper for marker in AMD_SHARED_MEMORY_APU_MARKERS) + + +def _normalize_apu_vram(name: str, vram_bytes: int) -> int: + if _is_shared_memory_apu(name) and vram_bytes < 2 * _GiB: + return 0 + return vram_bytes + + +def _make_gpu( + name: str, + *, + vram_bytes: int = 0, + rocm_version: str | None = None, +) -> GPUInfo: + shared_memory = _is_shared_memory_apu(name) + return GPUInfo( + name=name, + vendor="amd", + vram_bytes=_normalize_apu_vram(name, vram_bytes), + rocm_version=rocm_version, + memory_bandwidth_gbps=resolve_detected_bandwidth(name, vram_bytes), + shared_memory=shared_memory, + ) + + +_AMD_VENDOR_MARKERS = ( + "advanced micro devices", + "amd/ati", + "[amd]", + "[ati]", + "ati technologies", +) + + +def _vendor_is_amd(vendor: str) -> bool: + vendor_lower = vendor.lower() + return any(marker in vendor_lower for marker in _AMD_VENDOR_MARKERS) + + +def _detect_from_lspci() -> list[str]: + try: + result = subprocess.run( + ["lspci", "-mm"], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + logger.debug("lspci not available or timed out") + return [] + + if result.returncode != 0: + return [] + + names: list[str] = [] + seen: set[str] = set() + for line in result.stdout.splitlines(): + # `lspci -mm` is the machine-parsable format: + # "" "" "" [flags] ["" ""] + # Parse the quoted columns properly and check the vendor field + # specifically, instead of substring-matching the whole line (which + # would treat e.g. "Intel Corpor[ati]on" as AMD). + try: + tokens = shlex.split(line) + except ValueError: + continue + if len(tokens) < 4: + continue + device_class, vendor, device = tokens[1], tokens[2], tokens[3] + if device_class.lower() not in _DISPLAY_CLASSES: + continue + if not _vendor_is_amd(vendor): + continue + name = device.strip() or "AMD Graphics" + if name not in seen: + names.append(name) + seen.add(name) + return names + + +def _read_int(path: Path) -> int: + try: + text = path.read_text().strip() + except OSError: + return 0 + try: + return int(text, 0) + except ValueError: + return 0 + + +def _detect_from_sysfs(drm_path: Path = Path("/sys/class/drm")) -> list[GPUInfo]: + gpus: list[GPUInfo] = [] + seen: set[str] = set() + try: + cards = sorted(drm_path.glob("card[0-9]*")) + except OSError: + return [] + + for card in cards: + device = card / "device" + try: + vendor = (device / "vendor").read_text().strip().lower() + except OSError: + continue + if vendor != "0x1002": + continue + + name = "AMD Graphics" + try: + product_name = (device / "product_name").read_text().strip() + if product_name: + name = product_name + except OSError: + pass + + vram_bytes = _read_int(device / "mem_info_vram_total") + key = f"{name}:{vram_bytes}" + if key in seen: + continue + seen.add(key) + gpus.append(_make_gpu(name, vram_bytes=vram_bytes)) + return gpus + + +def _read_sysfs_amd_vram(drm_path: Path = Path("/sys/class/drm")) -> list[int]: + """Read VRAM for each AMD GPU from sysfs, in card order.""" + result: list[int] = [] + try: + cards = sorted(drm_path.glob("card[0-9]*")) + except OSError: + return [] + for card in cards: + device = card / "device" + try: + vendor = (device / "vendor").read_text().strip().lower() + except OSError: + continue + if vendor != "0x1002": + continue + result.append(_read_int(device / "mem_info_vram_total")) + return result + + +def _detect_amd_gpus_fallback() -> list[GPUInfo]: + # Prefer sysfs: it provides VRAM and sometimes a clean product name. + sysfs_gpus = _detect_from_sysfs() + + if sysfs_gpus: + # If sysfs names are generic ("AMD Graphics"), enrich with lspci names. + has_generic = any(g.name == "AMD Graphics" for g in sysfs_gpus) + if has_generic: + lspci_names = _detect_from_lspci() + if lspci_names and len(lspci_names) == len(sysfs_gpus): + return [ + _make_gpu( + lspci_names[i] if gpu.name == "AMD Graphics" else gpu.name, + vram_bytes=gpu.vram_bytes, + ) + for i, gpu in enumerate(sysfs_gpus) + ] + return sysfs_gpus + + # sysfs unavailable; fall back to lspci (name only), enriching with + # sysfs VRAM when possible. + names = _detect_from_lspci() + if names: + vram_list = _read_sysfs_amd_vram() + return [ + _make_gpu(name, vram_bytes=vram_list[i] if i < len(vram_list) else 0) + for i, name in enumerate(names) + ] + return [] + + +def detect_amd_gpus() -> list[GPUInfo]: + """Detect AMD GPUs. Returns empty list on failure.""" + gpus: list[GPUInfo] = [] + + # Get product names + try: + result = subprocess.run( + ["rocm-smi", "--showproductname", "--json"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + return _detect_amd_gpus_fallback() + product_data = json.loads(result.stdout) + except (FileNotFoundError, subprocess.TimeoutExpired, json.JSONDecodeError): + logger.debug("rocm-smi not available or failed") + return _detect_amd_gpus_fallback() + + # Get VRAM info + try: + result = subprocess.run( + ["rocm-smi", "--showmeminfo", "vram", "--json"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + return _detect_amd_gpus_fallback() + mem_data = json.loads(result.stdout) + except (FileNotFoundError, subprocess.TimeoutExpired, json.JSONDecodeError): + logger.debug("Failed to get AMD VRAM info") + return _detect_amd_gpus_fallback() + + # Get ROCm version + rocm_version = None + try: + result = subprocess.run( + ["rocm-smi", "--showdriverversion", "--json"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0: + driver_data = json.loads(result.stdout) + # Extract version from first card entry + for key, val in driver_data.items(): + if isinstance(val, dict) and "Driver version" in val: + rocm_version = val["Driver version"] + break + except Exception: + pass + + # Parse GPU info - rocm-smi JSON keys are like "card0", "card1" + for key in sorted(product_data.keys()): + if not key.startswith("card"): + continue + card_info = product_data[key] + # rocm-smi JSON uses "Card Series" (capital S) for the human-readable name. + # Fall back through SKU only as a last resort. + name = ( + card_info.get("Card Series") + or card_info.get("Card series") + or card_info.get("Card SKU") + or "Unknown AMD GPU" + ) + + vram_total = 0 + if key in mem_data: + vram_str = mem_data[key].get("VRAM Total Memory (B)", "0") + try: + vram_total = int(vram_str) + except (ValueError, TypeError): + pass + + gpus.append(_make_gpu(name, vram_bytes=vram_total, rocm_version=rocm_version)) + + return gpus diff --git a/src/whichllm/hardware/apple.py b/src/whichllm/hardware/apple.py new file mode 100644 index 0000000..47fbe54 --- /dev/null +++ b/src/whichllm/hardware/apple.py @@ -0,0 +1,131 @@ +"""Apple Silicon detection via system_profiler (macOS) and sysfs (Asahi Linux).""" + +from __future__ import annotations + +import json +import logging +import re +import subprocess +from pathlib import Path + +from whichllm.constants import GPU_BANDWIDTH +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + + +def _lookup_bandwidth(chip_name: str) -> float | None: + chip_upper = chip_name.upper() + for key in sorted(GPU_BANDWIDTH, key=len, reverse=True): + if key.upper() in chip_upper: + return GPU_BANDWIDTH[key] + return None + + +def detect_apple_gpu() -> list[GPUInfo]: + """Detect Apple Silicon GPU. Returns empty list on non-macOS or failure.""" + try: + result = subprocess.run( + ["system_profiler", "SPHardwareDataType", "-json"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + return [] + data = json.loads(result.stdout) + except (FileNotFoundError, subprocess.TimeoutExpired, json.JSONDecodeError): + logger.debug("system_profiler not available (not macOS)") + return [] + + try: + hw_items = data["SPHardwareDataType"] + hw = hw_items[0] + chip_name = hw.get("chip_type", "") + if not chip_name: + return [] + + # Apple Silicon uses unified memory - get total physical memory + memory_str = hw.get("physical_memory", "0 GB") + # Parse "32 GB" -> bytes + parts = memory_str.split() + mem_value = int(parts[0]) + mem_unit = parts[1].upper() if len(parts) > 1 else "GB" + multiplier = {"GB": 1024**3, "TB": 1024**4, "MB": 1024**2}.get( + mem_unit, 1024**3 + ) + unified_memory = mem_value * multiplier + + return [ + GPUInfo( + name=chip_name, + vendor="apple", + vram_bytes=unified_memory, # unified memory + memory_bandwidth_gbps=_lookup_bandwidth(chip_name), + shared_memory=True, + ) + ] + except (KeyError, IndexError, ValueError) as e: + logger.debug(f"Failed to parse Apple hardware info: {e}") + return [] + + +# ---- Asahi Linux (Apple Silicon on Linux) ---- + +_ASAHI_DRIVER_NAMES = ("asahi", "apple") + + +def _chip_name_from_devicetree() -> str | None: + """Extract Apple chip name from Linux device tree.""" + try: + raw = Path("/sys/firmware/devicetree/base/model").read_bytes() + model = raw.decode("utf-8", errors="replace").strip().rstrip("\x00") + if not model: + return None + m = re.search(r"\b(M\d+(?:\s+(?:Pro|Max|Ultra))?)\b", model) + if m: + return f"Apple {m.group(1)}" + return model + except OSError: + return None + + +def detect_apple_gpu_linux( + drm_path: Path = Path("/sys/class/drm"), +) -> list[GPUInfo]: + """Detect Apple Silicon GPU on Linux (Asahi driver). + + Returns empty list when no Asahi/Apple DRM device is found. + """ + try: + cards = sorted(drm_path.glob("card[0-9]*")) + except OSError: + return [] + + for card in cards: + driver = card / "device" / "driver" + try: + driver_name = driver.resolve().name + except OSError: + continue + if driver_name not in _ASAHI_DRIVER_NAMES: + continue + + chip_name = _chip_name_from_devicetree() or "Apple Silicon" + + # Unified memory — total system RAM is shared with the GPU. + import psutil + + unified_memory = psutil.virtual_memory().total + + return [ + GPUInfo( + name=chip_name, + vendor="apple", + vram_bytes=unified_memory, + memory_bandwidth_gbps=_lookup_bandwidth(chip_name), + shared_memory=True, + ) + ] + + return [] diff --git a/src/whichllm/hardware/cpu.py b/src/whichllm/hardware/cpu.py new file mode 100644 index 0000000..3d34b68 --- /dev/null +++ b/src/whichllm/hardware/cpu.py @@ -0,0 +1,225 @@ +"""CPU detection: name, cores, AVX2/AVX512 support.""" + +from __future__ import annotations + +import logging +import platform +import re +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _cpu_name_from_lscpu() -> str | None: + """Try to get CPU model name from lscpu (works on ARM/aarch64).""" + try: + result = subprocess.run(["lscpu"], capture_output=True, text=True, timeout=5) + if result.returncode == 0: + for line in result.stdout.splitlines(): + if line.strip().startswith("Model name"): + name = line.split(":", 1)[1].strip() + if name and name != "-": + return name + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + return None + + +def _cpu_name_from_devicetree() -> str | None: + """Extract CPU/chip name from device tree (ARM Linux, Asahi).""" + try: + raw = Path("/sys/firmware/devicetree/base/model").read_bytes() + model = raw.decode("utf-8", errors="replace").strip().rstrip("\x00") + if not model: + return None + # "Apple MacBook Air (M2, 2022)" → "Apple M2" + m = re.search(r"\b(M\d+(?:\s+(?:Pro|Max|Ultra))?)\b", model) + if m: + return f"Apple {m.group(1)}" + return model + except OSError: + return None + + +def _clean_cpu_name(name: str | None) -> str | None: + if name is None: + return None + cleaned = name.strip() + if not cleaned or cleaned == "-" or cleaned.lower() == "name": + return None + return cleaned + + +def _cpu_name_from_wmic() -> str | None: + try: + result = subprocess.run( + ["wmic", "cpu", "get", "name"], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.SubprocessError, OSError): + return None + + if result.returncode != 0: + return None + + for line in result.stdout.splitlines(): + name = _clean_cpu_name(line) + if name: + return name + return None + + +def _cpu_name_from_windows_cim() -> str | None: + script = ( + "Get-CimInstance Win32_Processor | Select-Object -First 1 -ExpandProperty Name" + ) + for executable in ("powershell", "pwsh"): + try: + result = subprocess.run( + [executable, "-NoProfile", "-Command", script], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.SubprocessError, OSError): + continue + + if result.returncode != 0: + continue + + for line in result.stdout.splitlines(): + name = _clean_cpu_name(line) + if name: + return name + return None + + +def detect_cpu_name() -> str: + """Get CPU model name.""" + system = platform.system() + try: + if system == "Linux": + with open("/proc/cpuinfo") as f: + for line in f: + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + # ARM/aarch64: /proc/cpuinfo has no model name field. + # Try lscpu, then device tree. + name = _cpu_name_from_lscpu() or _cpu_name_from_devicetree() + if name: + return name + elif system == "Darwin": + result = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + name = _clean_cpu_name(result.stdout) + if name: + return name + elif system == "Windows": + name = _cpu_name_from_wmic() or _cpu_name_from_windows_cim() + if name: + return name + except Exception as e: + logger.debug(f"Failed to detect CPU name: {e}") + return "Unknown CPU" + + +def _count_physical_cores_linux() -> int | None: + """Count unique physical cores from /proc/cpuinfo (handles WSL2).""" + try: + physical_ids: set[tuple[str, str]] = set() + current_physical = "" + current_core = "" + with open("/proc/cpuinfo") as f: + for line in f: + if line.startswith("physical id"): + current_physical = line.split(":", 1)[1].strip() + elif line.startswith("core id"): + current_core = line.split(":", 1)[1].strip() + physical_ids.add((current_physical, current_core)) + if physical_ids: + return len(physical_ids) + except Exception: + pass + return None + + +def detect_cpu_cores() -> int: + """Get number of physical CPU cores.""" + import psutil + + cores = psutil.cpu_count(logical=False) + if cores: + return cores + + # Fallback for WSL2 where psutil may return None for physical cores + if platform.system() == "Linux": + linux_cores = _count_physical_cores_linux() + if linux_cores: + return linux_cores + + return psutil.cpu_count(logical=True) or 1 + + +def _detect_avx_linux() -> tuple[bool, bool]: + """Detect AVX2/AVX512 on Linux via /proc/cpuinfo.""" + has_avx2 = False + has_avx512 = False + try: + with open("/proc/cpuinfo") as f: + content = f.read() + flags_line = "" + for line in content.split("\n"): + if line.startswith("flags"): + flags_line = line + break + has_avx2 = "avx2" in flags_line + has_avx512 = "avx512f" in flags_line + except Exception: + pass + return has_avx2, has_avx512 + + +def _detect_avx_darwin() -> tuple[bool, bool]: + """Detect AVX2/AVX512 on macOS via sysctl.""" + has_avx2 = False + has_avx512 = False + try: + result = subprocess.run( + ["sysctl", "-n", "hw.optional.avx2_0"], + capture_output=True, + text=True, + timeout=5, + ) + has_avx2 = result.stdout.strip() == "1" + except Exception: + pass + try: + result = subprocess.run( + ["sysctl", "-n", "hw.optional.avx512f"], + capture_output=True, + text=True, + timeout=5, + ) + has_avx512 = result.stdout.strip() == "1" + except Exception: + pass + return has_avx2, has_avx512 + + +def detect_avx_support() -> tuple[bool, bool]: + """Detect AVX2 and AVX512 support. Returns (has_avx2, has_avx512).""" + system = platform.system() + if system == "Linux": + return _detect_avx_linux() + elif system == "Darwin": + return _detect_avx_darwin() + # Windows / fallback: assume AVX2 on modern CPUs + return True, False diff --git a/src/whichllm/hardware/detector.py b/src/whichllm/hardware/detector.py new file mode 100644 index 0000000..da4cf80 --- /dev/null +++ b/src/whichllm/hardware/detector.py @@ -0,0 +1,56 @@ +"""Unified hardware detection orchestrator.""" + +from __future__ import annotations + +import logging +import platform + +from whichllm.hardware.amd import detect_amd_gpus +from whichllm.hardware.apple import detect_apple_gpu, detect_apple_gpu_linux +from whichllm.hardware.cpu import detect_avx_support, detect_cpu_cores, detect_cpu_name +from whichllm.hardware.intel import detect_intel_gpus +from whichllm.hardware.memory import detect_disk_free_bytes, detect_ram_bytes +from whichllm.hardware.nvidia import detect_nvidia_gpus +from whichllm.hardware.types import HardwareInfo +from whichllm.hardware.windows import detect_windows_gpus + +logger = logging.getLogger(__name__) + + +def detect_hardware() -> HardwareInfo: + """Detect all hardware. Each detector is fail-safe (returns empty on error).""" + os_name = platform.system().lower() + if os_name not in ("linux", "darwin", "windows"): + os_name = "linux" + + # GPU detection + gpus = [] + gpus.extend(detect_nvidia_gpus()) + if os_name == "linux": + gpus.extend(detect_amd_gpus()) + gpus.extend(detect_intel_gpus()) + gpus.extend(detect_apple_gpu_linux()) + if os_name == "darwin": + gpus.extend(detect_apple_gpu()) + if os_name == "windows": + gpus.extend(detect_windows_gpus()) + + # CPU + cpu_name = detect_cpu_name() + cpu_cores = detect_cpu_cores() + has_avx2, has_avx512 = detect_avx_support() + + # Memory + ram_bytes = detect_ram_bytes() + disk_free = detect_disk_free_bytes() + + return HardwareInfo( + gpus=gpus, + cpu_name=cpu_name, + cpu_cores=cpu_cores, + has_avx2=has_avx2, + has_avx512=has_avx512, + ram_bytes=ram_bytes, + disk_free_bytes=disk_free, + os=os_name, + ) diff --git a/src/whichllm/hardware/gpu_db.py b/src/whichllm/hardware/gpu_db.py new file mode 100644 index 0000000..e679615 --- /dev/null +++ b/src/whichllm/hardware/gpu_db.py @@ -0,0 +1,219 @@ +"""Resolve GPU memory bandwidth for *detected* hardware. + +Detection passes the raw driver name (e.g. ``"NVIDIA GeForce RTX 5090 Laptop +GPU"``). Unlike ``--gpu`` simulation, where the user typed the name and a fuzzy +guess plus a ``(simulated)`` label is acceptable, a wrong match on real +hardware is worse than no data: giving a laptop card its desktop bandwidth +produces confidently wrong speed estimates and oversized recommendations +(issues #74, #61, #93). + +So this resolver is deliberately strict: + +* The hand-curated ``GPU_BANDWIDTH`` table stays authoritative and is tried + first, so existing behaviour for known cards is unchanged. +* The curated lookup is mobile-aware: a laptop/Max-Q driver name will not match + a desktop key (``"RTX 5090"`` no longer swallows ``"RTX 5090 Laptop GPU"``). +* dbgpu is only consulted to fill gaps, and only via an exact normalised-name + hit or a name plus a VRAM-size suffix (``"RTX 4060 Ti 16 GB"``). It never + falls back to fuzzy matching, so a variant qualifier (Ti / SUPER / Mobile / + Max-Q / XT) can never be silently dropped onto the wrong card. +* ``"Laptop GPU"`` in a driver name is normalised to dbgpu's ``"Mobile"``. + +The change is purely additive: cards already covered by ``GPU_BANDWIDTH`` keep +their exact value, and cards that previously resolved to ``None`` now get a +correct bandwidth whenever dbgpu can identify them safely. +""" + +from __future__ import annotations + +import functools +import logging +import re + +from whichllm.constants import GPU_BANDWIDTH, GPU_MEMORY_CLOCK_VARIANTS, _GiB + +logger = logging.getLogger(__name__) + +_TRADEMARK_RE = re.compile(r"\((?:tm|r)\)", re.IGNORECASE) +_VENDOR_WORD_RE = re.compile(r"\b(?:nvidia|amd|ati|intel|corporation)\b", re.IGNORECASE) +_LAPTOP_GPU_RE = re.compile(r"\blaptop gpu\b", re.IGNORECASE) +_TRAILING_GRAPHICS_RE = re.compile(r"\bgraphics\s*$", re.IGNORECASE) +_WHITESPACE_RE = re.compile(r"\s+") +_MOBILE_MARKER_RE = re.compile(r"\b(?:laptop|mobile|max-?q)\b", re.IGNORECASE) +# Driver names write VRAM bins without a space ("RTX A2000 12GB"); dbgpu +# writes "RTX A2000 12 GB" (#98). +_VRAM_NOSPACE_RE = re.compile(r"\b(\d+)GB\b", re.IGNORECASE) +_BRACKET_RE = re.compile(r"\[(.+)]") +# A dbgpu name may extend a matched query with a VRAM-size bin only +# ("RTX 4060 Ti 16 GB"). A variant word (Mobile, Ti, D, ...) is never benign. +_VRAM_SUFFIX_RE = re.compile(r"^\s+\d+\s*gb\b", re.IGNORECASE) +_VRAM_GB_RE = re.compile(r"(\d+)\s*gb", re.IGNORECASE) + + +def _normalize_detected_name(name: str) -> str: + """Reduce a raw driver name toward dbgpu's naming convention.""" + text = _TRADEMARK_RE.sub("", name) + text = _VENDOR_WORD_RE.sub("", text) + text = _LAPTOP_GPU_RE.sub("Mobile", text) + text = _TRAILING_GRAPHICS_RE.sub("", text) + text = _VRAM_NOSPACE_RE.sub(r"\1 GB", text) + return _WHITESPACE_RE.sub(" ", text).strip() + + +_SORTED_BW_KEYS = sorted(GPU_BANDWIDTH, key=len, reverse=True) + + +def _substring_bandwidth(name: str) -> float | None: + """Curated ``GPU_BANDWIDTH`` lookup (longest key first), mobile-aware. + + When the detected name is a laptop/Max-Q card, a desktop key is not allowed + to match it via substring, since the two have very different bandwidth. + """ + if not name: + return None + name_upper = name.upper() + name_is_mobile = bool(_MOBILE_MARKER_RE.search(name)) + for key in _SORTED_BW_KEYS: + if key.upper() in name_upper: + if name_is_mobile and not _MOBILE_MARKER_RE.search(key): + continue + return GPU_BANDWIDTH[key] + return None + + +def _static_bandwidth(name: str) -> float | None: + """Curated lookup that also handles compound lspci names. + + Compound names like ``"Navi 22 [Radeon RX 6700/6700 XT/6750 XT / + 6800M/6850M XT]"`` list several variants; the first segment that resolves + wins, with a ``"RX "`` prefix retry for bare segments like ``"6750 XT"``. + dbgpu is never consulted for these: the name does not identify a single + card, so the curated value for the listed family is the safest answer. + """ + if not name: + return None + if "/" not in name: + bandwidth = _substring_bandwidth(name) + if bandwidth is not None: + return bandwidth + normalized = _normalize_detected_name(name) + return _substring_bandwidth(normalized) if normalized != name else None + bracket = _BRACKET_RE.search(name) + raw = bracket.group(1) if bracket else name + for seg in raw.split("/"): + seg = seg.strip() + if not seg: + continue + bandwidth = _substring_bandwidth(seg) or _substring_bandwidth(f"RX {seg}") + if bandwidth is not None: + return bandwidth + return None + + +def _vram_gb(canonical_name: str) -> int | None: + match = _VRAM_GB_RE.search(canonical_name) + return int(match.group(1)) if match else None + + +@functools.lru_cache(maxsize=1) +def _dbgpu_index() -> tuple[object | None, dict[str, str] | None]: + """Build ``{normalized_name: canonical_dbgpu_name}``. + + Returns ``(db, index)`` or ``(None, None)`` if dbgpu is unavailable, so the + resolver degrades to static-only instead of raising. + """ + try: + from dbgpu import GPUDatabase + + db = GPUDatabase.default() + except Exception as exc: # pragma: no cover - dbgpu is a hard dependency + logger.debug("dbgpu unavailable, using static bandwidth only: %s", exc) + return None, None + index: dict[str, str] = {} + for canonical in db.names: + index.setdefault(_normalize_detected_name(canonical).lower(), canonical) + return db, index + + +def _dbgpu_bandwidth(name: str, vram_bytes: int | None) -> float | None: + """Strict dbgpu bandwidth lookup. Never fuzzy-matches a variant away.""" + db, index = _dbgpu_index() + if db is None or index is None: + return None + query = _normalize_detected_name(name).lower() + if not query: + return None + + if query in index: + candidates = [index[query]] + else: + candidates = [ + original + for normalized, original in index.items() + if normalized.startswith(query + " ") + and _VRAM_SUFFIX_RE.match(normalized[len(query) :]) + ] + if not candidates: + return None + if vram_bytes and len(candidates) > 1: + target_gb = round(vram_bytes / _GiB) + same_vram = [c for c in candidates if _vram_gb(c) == target_gb] + if same_vram: + candidates = same_vram + + # If several VRAM bins remain (VRAM unknown or no bin matched it), take the + # lowest bandwidth among them: an ambiguous match must never over-promise + # speed. Bins of the same card usually share one value anyway. + bandwidths: list[float] = [] + for canonical in candidates: + try: + spec = db[canonical] + except KeyError: # pragma: no cover - canonical comes from the index + continue + bandwidth = getattr(spec, "memory_bandwidth_gb_s", None) + if bandwidth: + bandwidths.append(float(bandwidth)) + return min(bandwidths) if bandwidths else None + + +def _memory_clock_variant_bandwidth( + name: str, mem_clock_mhz: float | None +) -> float | None: + """Disambiguate cards sold in multiple memory types by max memory clock. + + Some GPUs (e.g. GTX 1650 GDDR5 vs GDDR6) share a marketing name and PCI + device id, so only the memory clock tells them apart. Returns the matching + variant bandwidth when the name is a known dual-memory card and the clock is + known, else ``None`` (so the caller falls back to the curated default). + """ + if not name or not mem_clock_mhz or mem_clock_mhz <= 0: + return None + name_upper = name.upper() + for key in sorted(GPU_MEMORY_CLOCK_VARIANTS, key=len, reverse=True): + if key.upper() in name_upper: + for min_clock, bandwidth in GPU_MEMORY_CLOCK_VARIANTS[key]: + if mem_clock_mhz >= min_clock: + return bandwidth + return None + + +def resolve_detected_bandwidth( + name: str, + vram_bytes: int | None = None, + mem_clock_mhz: float | None = None, +) -> float | None: + """Best memory bandwidth (GB/s) for a detected GPU, or ``None`` if unknown. + + Memory-clock variant disambiguation wins first (for cards that share a name + across memory types, e.g. GTX 1650 GDDR5/GDDR6); then curated + ``GPU_BANDWIDTH``; then dbgpu fills the gaps. ``vram_bytes`` (when known) + disambiguates same-name cards that ship in multiple VRAM bins; + ``mem_clock_mhz`` (max memory clock, when known) disambiguates memory-type + variants. With both unset, behaviour is identical to before. + """ + if not name: + return None + variant = _memory_clock_variant_bandwidth(name, mem_clock_mhz) + if variant is not None: + return variant + return _static_bandwidth(name) or _dbgpu_bandwidth(name, vram_bytes) diff --git a/src/whichllm/hardware/gpu_simulator.py b/src/whichllm/hardware/gpu_simulator.py new file mode 100644 index 0000000..e45ccde --- /dev/null +++ b/src/whichllm/hardware/gpu_simulator.py @@ -0,0 +1,346 @@ +"""Create synthetic GPUInfo for GPU simulation (--gpu flag). + +Uses the dbgpu package (2000+ GPU database from TechPowerUp) for dynamic +lookup of VRAM, memory bandwidth, and compute capability. +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dbgpu import GPUSpecification + +from whichllm.constants import ( + AMD_SHARED_MEMORY_APU_MARKERS, + CURATED_GPU_SPECS, + GPU_BANDWIDTH, + CuratedGPUSpec, + _GiB, +) +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + +_MANUFACTURER_TO_VENDOR: dict[str, str] = { + "NVIDIA": "nvidia", + "AMD": "amd", + "ATI": "amd", + "Intel": "intel", + "Apple": "apple", +} + +_MANUFACTURER_PREFIXES = ["GeForce ", "Radeon ", "Arc ", "NVIDIA ", "AMD "] +_COMMON_GPU_ALIASES: dict[str, list[str]] = { + "a10080gb": [ + "NVIDIA A100 PCIe 80 GB", + "NVIDIA A100 SXM4 80 GB", + ], + "h10080gb": [ + "NVIDIA H100 PCIe 80 GB", + "NVIDIA H100 SXM5 80 GB", + ], +} + +# Apple Silicon chips. dbgpu does not include these (it tracks discrete GPUs +# via TechPowerUp), but users routinely simulate with --gpu "M2 Max" etc. +# Without this short-circuit, "M1" fuzzy-matches the 1997 ATI Rage Mobility-M1 +# and "M3 Max" falls through to vendor=nvidia default. +# Format: chip_name -> (canonical_name, default_unified_memory_gb). +# --vram override is still respected; the default is the most common SKU. +_APPLE_SILICON_CHIPS: dict[str, tuple[str, float]] = { + "M1": ("Apple M1", 8.0), + "M1 Pro": ("Apple M1 Pro", 16.0), + "M1 Max": ("Apple M1 Max", 32.0), + "M1 Ultra": ("Apple M1 Ultra", 64.0), + "M2": ("Apple M2", 16.0), + "M2 Pro": ("Apple M2 Pro", 16.0), + "M2 Max": ("Apple M2 Max", 32.0), + "M2 Ultra": ("Apple M2 Ultra", 64.0), + "M3": ("Apple M3", 16.0), + "M3 Pro": ("Apple M3 Pro", 18.0), + "M3 Max": ("Apple M3 Max", 36.0), + "M3 Ultra": ("Apple M3 Ultra", 96.0), + "M4": ("Apple M4", 16.0), + "M4 Pro": ("Apple M4 Pro", 24.0), + "M4 Max": ("Apple M4 Max", 36.0), + "M4 Ultra": ("Apple M4 Ultra", 64.0), + "M5": ("Apple M5", 16.0), + "M5 Pro": ("Apple M5 Pro", 24.0), + "M5 Max": ("Apple M5 Max", 36.0), +} + + +def _lookup_apple_silicon( + name: str, +) -> tuple[str, str, float, float] | None: + """Match Apple Silicon chip names. Returns (canonical_name, vendor, + default_vram_gb, bandwidth_gbps) or None. + + Matches are case-insensitive and accept "M2 Max", "m2max", and + display-name forms such as "Apple M2 Max". Longest match wins so + "M2 Ultra" does not get caught by the "M2" entry. + """ + compact = re.sub(r"\s+", "", name).lower() + if compact.startswith("apple"): + compact = compact.removeprefix("apple") + + # Sort keys by length descending so "M2 Ultra" wins over "M2". + for key in sorted(_APPLE_SILICON_CHIPS, key=len, reverse=True): + key_compact = re.sub(r"\s+", "", key).lower() + if compact == key_compact: + canonical, default_vram = _APPLE_SILICON_CHIPS[key] + bandwidth = GPU_BANDWIDTH.get(key, 100.0) + return canonical, "apple", default_vram, bandwidth + return None + + +def _is_amd_shared_memory_apu(name: str) -> bool: + name_upper = name.upper() + return any(marker in name_upper for marker in AMD_SHARED_MEMORY_APU_MARKERS) + + +def _lookup_static_bandwidth(name: str) -> float | None: + name_upper = name.upper() + for key in sorted(GPU_BANDWIDTH, key=len, reverse=True): + if key.upper() in name_upper: + return GPU_BANDWIDTH[key] + return None + + +def _lookup_curated_spec(name: str) -> CuratedGPUSpec | None: + name_upper = name.upper() + for key in sorted(CURATED_GPU_SPECS, key=len, reverse=True): + if key.upper() in name_upper: + return CURATED_GPU_SPECS[key] + return None + + +def _normalize_gpu_name(name: str) -> str: + """Normalize user input: 'GTX1080' → 'GTX 1080', 'RX7900XTX' → 'RX 7900 XTX'.""" + # Insert space between letters and digits + name = re.sub(r"([A-Za-z])(\d)", r"\1 \2", name) + # Insert space between digits and letters + name = re.sub(r"(\d)([A-Za-z])", r"\1 \2", name) + # Collapse multiple spaces + return re.sub(r"\s+", " ", name).strip() + + +def _substring_search(db, name: str): + """Substring match with word-boundary filtering. + + e.g. "RTX 3060" should match "GeForce RTX 3060 12 GB" but NOT "GeForce RTX 3060 Ti". + """ + name_upper = name.upper() + candidates = [] + for db_name in db.names: + idx = db_name.upper().find(name_upper) + if idx < 0: + continue + after = db_name[idx + len(name) :] + # Accept if nothing follows, or what follows is VRAM/form-factor spec + # Reject if a variant suffix follows (Ti, SUPER, Mobile, Max-Q, etc.) + if not after or re.match(r"^(\s+(\d|GA\d|PCIe|SXM|NVL|CNX))", after): + candidates.append(db_name) + if candidates: + candidates.sort(key=len) + return db[candidates[0]] + return None + + +def _lookup_dbgpu(name: str) -> GPUSpecification | None: + """Look up GPU spec from dbgpu database. Returns GPUSpecification or None.""" + from dbgpu import GPUDatabase + + db = GPUDatabase.default() + + # Normalize input: "GTX1080" → "GTX 1080" + normalized = _normalize_gpu_name(name) + compact = re.sub(r"\s+", "", normalized.lower()) + names_to_try = [name] if normalized == name else [name, normalized] + alias_hits = _COMMON_GPU_ALIASES.get(compact) + if alias_hits: + names_to_try.extend(alias_hits) + + for n in names_to_try: + # 1) Exact key lookup + try: + return db[n] + except KeyError: + pass + + # 2) Try with common manufacturer prefixes + for prefix in _MANUFACTURER_PREFIXES: + try: + return db[prefix + n] + except KeyError: + pass + + # 3) Substring match with word-boundary filtering + result = _substring_search(db, n) + if result is not None: + return result + + # 4) Fuzzy search as last resort (use normalized name + token_set_ratio) + try: + from thefuzz import fuzz, process + + results = process.extract( + normalized, db.names, limit=3, scorer=fuzz.token_set_ratio + ) + if results and results[0][1] >= 90: + return db[results[0][0]] + # Store top suggestions for error messages + if results: + _last_suggestions[:] = [(n, s) for n, s in results if s >= 70] + except ImportError: + pass + return None + + +# Mutable list to pass suggestions from lookup to error message +_last_suggestions: list[tuple[str, int]] = [] + + +def parse_synthetic_gpu_specs(values: Sequence[str] | str) -> list[str]: + """Expand CLI GPU simulation values into individual GPU names. + + Accepts repeated options, comma-separated names, and count shorthand such + as ``2x RTX 4090``. The returned names are still looked up by + ``create_synthetic_gpu`` so existing fuzzy matching and aliases stay in + one place. + """ + raw_values = [values] if isinstance(values, str) else list(values) + gpu_names: list[str] = [] + + for raw in raw_values: + for part in raw.split(","): + spec = part.strip() + if not spec: + raise ValueError("Empty GPU entry in --gpu.") + + count_match = re.match(r"^(\d+)\s*x\s+(.+)$", spec, re.IGNORECASE) + if count_match: + count = int(count_match.group(1)) + name = count_match.group(2).strip() + if count < 1: + raise ValueError("GPU count must be at least 1.") + if not name: + raise ValueError("GPU count shorthand requires a GPU name.") + gpu_names.extend([name] * count) + else: + gpu_names.append(spec) + + if not gpu_names: + raise ValueError("At least one GPU must be specified.") + return gpu_names + + +def create_synthetic_gpus( + values: Sequence[str] | str, + vram_override_gb: float | None = None, +) -> list[GPUInfo]: + """Create one or more synthetic GPUs from CLI-style values.""" + names = parse_synthetic_gpu_specs(values) + if vram_override_gb is not None and len(names) != 1: + raise ValueError( + "--vram currently supports exactly one simulated GPU. " + "For multi-GPU simulation, specify known GPU names and omit --vram." + ) + return [create_synthetic_gpu(name, vram_override_gb) for name in names] + + +def create_synthetic_gpu(name: str, vram_override_gb: float | None = None) -> GPUInfo: + """Create a synthetic GPUInfo from a GPU name. + + Looks up specs from the dbgpu database (2000+ GPUs). + + Args: + name: GPU name (e.g. "RTX 4090", "RX 7900 XTX"). + vram_override_gb: Override VRAM in GB. Required if GPU not in database. + + Returns: + GPUInfo with ``(simulated)`` suffix in the name. + + Raises: + ValueError: If GPU is not found and no vram_override_gb given. + """ + _last_suggestions.clear() + + amd_shared_memory_apu = _is_amd_shared_memory_apu(name) + curated = _lookup_curated_spec(name) + + # Apple Silicon short-circuit: dbgpu has no Apple entries, so we check + # first to avoid fuzzy-matching "M1" against "Rage Mobility-M1". + apple_hit = _lookup_apple_silicon(name) + if apple_hit is not None: + canonical, vendor, default_vram_gb, bandwidth = apple_hit + vram_gb = vram_override_gb if vram_override_gb is not None else default_vram_gb + return GPUInfo( + name=f"{canonical} (simulated)", + vendor=vendor, + vram_bytes=int(vram_gb * _GiB), + memory_bandwidth_gbps=bandwidth, + shared_memory=True, + vram_overridden=vram_override_gb is not None, + ) + + spec = _lookup_dbgpu(name) + + # VRAM + if vram_override_gb is not None: + vram_bytes = int(vram_override_gb * _GiB) + elif spec is not None and spec.memory_size_gb: + vram_bytes = int(spec.memory_size_gb * _GiB) + elif curated is not None: + vram_bytes = int(curated.vram_gb * _GiB) + else: + msg = f"Unknown GPU '{name}'." + if _last_suggestions: + candidates = ", ".join(n for n, _ in _last_suggestions) + msg += f" Did you mean: {candidates}?" + msg += " Use --vram to specify VRAM in GB." + raise ValueError(msg) + + # Bandwidth + bandwidth: float | None = None + if spec is not None and spec.memory_bandwidth_gb_s: + bandwidth = spec.memory_bandwidth_gb_s + if bandwidth is None and curated is not None: + bandwidth = curated.memory_bandwidth_gbps + if bandwidth is None: + bandwidth = _lookup_static_bandwidth(name) + + # Compute capability (CUDA version in dbgpu = compute capability) + compute_cap: tuple[int, int] | None = None + if spec is not None and spec.cuda_major_version is not None: + compute_cap = (spec.cuda_major_version, spec.cuda_minor_version or 0) + + # Vendor + vendor = "nvidia" + if spec is not None: + vendor = _MANUFACTURER_TO_VENDOR.get(spec.manufacturer, "nvidia") + elif curated is not None: + vendor = curated.vendor + elif amd_shared_memory_apu: + vendor = "amd" + + if spec is not None: + display_name = spec.name + elif curated is not None: + display_name = curated.name + else: + display_name = name + + return GPUInfo( + name=f"{display_name} (simulated)", + vendor=vendor, + vram_bytes=vram_bytes, + compute_capability=compute_cap, + memory_bandwidth_gbps=bandwidth, + shared_memory=curated.shared_memory if curated else amd_shared_memory_apu, + vram_overridden=vram_override_gb is not None, + ) diff --git a/src/whichllm/hardware/intel.py b/src/whichllm/hardware/intel.py new file mode 100644 index 0000000..127a496 --- /dev/null +++ b/src/whichllm/hardware/intel.py @@ -0,0 +1,136 @@ +"""Intel integrated GPU detection on Linux.""" + +from __future__ import annotations + +import logging +import subprocess +from pathlib import Path + +from whichllm.constants import ( + CURATED_GPU_SPECS, + INTEL_PCI_DEVICE_NAMES, + CuratedGPUSpec, + _GiB, +) +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + + +_DISPLAY_CLASSES = ( + "vga compatible controller", + "3d controller", + "display controller", +) + + +def _normalize_lspci_name(line: str) -> str: + parts = [p.strip() for p in line.split('"') if p.strip() and p.strip() != "\t"] + for i, part in enumerate(parts): + if part.lower() == "intel corporation" and i + 1 < len(parts): + return parts[i + 1] + return "Intel Integrated Graphics" + + +def _detect_from_lspci() -> list[str]: + try: + result = subprocess.run( + ["lspci", "-mm"], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + logger.debug("lspci not available or timed out") + return [] + + if result.returncode != 0: + return [] + + names: list[str] = [] + seen: set[str] = set() + for line in result.stdout.splitlines(): + line_lower = line.lower() + if "intel" not in line_lower or not any( + display_class in line_lower for display_class in _DISPLAY_CLASSES + ): + continue + name = _normalize_lspci_name(line) + if name not in seen: + names.append(name) + seen.add(name) + return names + + +def _detect_from_sysfs(drm_path: Path = Path("/sys/class/drm")) -> list[str]: + names: list[str] = [] + seen: set[str] = set() + try: + cards = sorted(drm_path.glob("card[0-9]*")) + except OSError: + return [] + + for card in cards: + device = card / "device" + try: + vendor = (device / "vendor").read_text().strip().lower() + except OSError: + continue + if vendor != "0x8086": + continue + + name = "Intel Integrated Graphics" + known_device = False + try: + device_id = (device / "device").read_text().strip().lower() + mapped_name = INTEL_PCI_DEVICE_NAMES.get(device_id) + if mapped_name: + name = mapped_name + known_device = True + except OSError: + pass + + try: + product_name = (device / "product_name").read_text().strip() + if product_name and not known_device: + name = product_name + except OSError: + pass + + if name not in seen: + names.append(name) + seen.add(name) + return names + + +def _lookup_curated_spec(name: str) -> CuratedGPUSpec | None: + name_upper = name.upper() + for key in sorted(CURATED_GPU_SPECS, key=len, reverse=True): + if key.upper() in name_upper: + return CURATED_GPU_SPECS[key] + return None + + +def _gpu_info_from_name(name: str) -> GPUInfo: + curated = _lookup_curated_spec(name) + if curated is not None: + return GPUInfo( + name=name, + vendor=curated.vendor, + vram_bytes=int(curated.vram_gb * _GiB), + memory_bandwidth_gbps=curated.memory_bandwidth_gbps, + shared_memory=curated.shared_memory, + ) + return GPUInfo( + name=name, + vendor="intel", + vram_bytes=0, + shared_memory=True, + ) + + +def detect_intel_gpus() -> list[GPUInfo]: + """Detect Linux Intel iGPUs. Returns empty list on failure.""" + names = _detect_from_lspci() or _detect_from_sysfs() + + return [_gpu_info_from_name(name) for name in names] diff --git a/src/whichllm/hardware/memory.py b/src/whichllm/hardware/memory.py new file mode 100644 index 0000000..9f16ba9 --- /dev/null +++ b/src/whichllm/hardware/memory.py @@ -0,0 +1,52 @@ +"""RAM and disk space detection.""" + +from __future__ import annotations + +import os +import shutil + +import psutil + + +def detect_ram_bytes() -> int: + """Get total physical RAM in bytes.""" + return psutil.virtual_memory().total + + +def detect_available_ram_bytes() -> int: + """Get currently available RAM bytes.""" + return psutil.virtual_memory().available + + +def estimate_usable_ram(total: int) -> int: + """Estimate RAM available for model loading after OS/background reserve. + + Uses a bounded-reserve formula: total - clamp(total * 0.15, 4 GiB, 32 GiB). + """ + _GiB = 1024**3 + reserve = int(total * 0.15) + reserve = max(4 * _GiB, min(reserve, 32 * _GiB)) + return max(0, total - reserve) + + +def effective_usable_ram(total: int, budget: int | None = None) -> int: + """Estimate usable RAM, optionally capped by a user/runtime budget.""" + usable = estimate_usable_ram(total) + if budget is None: + return usable + return max(0, min(usable, budget)) + + +def detect_disk_free_bytes(path: str | None = None) -> int: + """Get free disk space in bytes at the given path. + + Defaults to the user's home directory, which is more accurate + on macOS where / may be a read-only system volume. + """ + if path is None: + path = os.path.expanduser("~") + try: + usage = shutil.disk_usage(path) + return usage.free + except OSError: + return 0 diff --git a/src/whichllm/hardware/nvidia.py b/src/whichllm/hardware/nvidia.py new file mode 100644 index 0000000..785c131 --- /dev/null +++ b/src/whichllm/hardware/nvidia.py @@ -0,0 +1,198 @@ +"""NVIDIA GPU detection via NVML with nvidia-smi fallback.""" + +from __future__ import annotations + +import logging +import re +import subprocess + +from whichllm.constants import NVIDIA_COMPUTE_CAPABILITY, _GiB +from whichllm.hardware.gpu_db import _static_bandwidth, resolve_detected_bandwidth +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + +_NVIDIA_UNIFIED_MEMORY_MARKERS = ("GB10", "DGX SPARK") + + +def _lookup_compute_capability(name: str) -> tuple[int, int] | None: + name_upper = name.upper() + for key, cc in NVIDIA_COMPUTE_CAPABILITY.items(): + if key.upper() in name_upper: + return cc + return None + + +def _lookup_bandwidth(name: str) -> float | None: + """Curated GPU_BANDWIDTH lookup. Kept for regression tests; live detection + goes through ``resolve_detected_bandwidth``, which also consults dbgpu.""" + return _static_bandwidth(name) + + +def _is_unified_memory_nvidia_gpu(name: str) -> bool: + name_upper = name.upper() + return any(marker in name_upper for marker in _NVIDIA_UNIFIED_MEMORY_MARKERS) + + +def _system_memory_bytes() -> int: + from whichllm.hardware.memory import detect_ram_bytes + + ram_bytes = detect_ram_bytes() + if ram_bytes > 0: + return ram_bytes + return 128 * _GiB + + +def _make_nvidia_gpu( + name: str, + vram_bytes: int | None, + cuda_version: str | None = None, + mem_clock_mhz: float | None = None, +) -> GPUInfo: + shared_memory = _is_unified_memory_nvidia_gpu(name) + if shared_memory and (vram_bytes is None or vram_bytes <= 0): + vram_bytes = _system_memory_bytes() + elif vram_bytes is None: + vram_bytes = 0 + + return GPUInfo( + name=name, + vendor="nvidia", + vram_bytes=vram_bytes, + compute_capability=_lookup_compute_capability(name), + cuda_version=cuda_version, + memory_bandwidth_gbps=resolve_detected_bandwidth( + name, vram_bytes, mem_clock_mhz + ), + shared_memory=shared_memory, + ) + + +def _run_smi_query(fields: str) -> str: + result = subprocess.run( + ["nvidia-smi", f"--query-gpu={fields}", "--format=csv,noheader,nounits"], + capture_output=True, + check=True, + text=True, + timeout=5, + ) + return result.stdout + + +def _detect_nvidia_gpus_via_smi() -> list[GPUInfo]: + """Detect NVIDIA GPUs using nvidia-smi when Python NVML cannot load. + + The max memory clock (used to disambiguate same-name memory variants such as + GTX 1650 GDDR5/GDDR6) is queried opportunistically. If the 3-field query + fails on a driver/card that rejects ``clocks.max.memory``, we retry the + original 2-field query so a missing optional field never wipes out detection. + """ + try: + stdout = _run_smi_query("name,memory.total,clocks.max.memory") + except (FileNotFoundError, subprocess.SubprocessError, OSError) as e: + logger.debug(f"nvidia-smi 3-field query failed ({e}); retrying without clock") + try: + stdout = _run_smi_query("name,memory.total") + except (FileNotFoundError, subprocess.SubprocessError, OSError) as e2: + logger.debug(f"nvidia-smi fallback failed: {e2}") + return [] + + gpus: list[GPUInfo] = [] + for line in stdout.splitlines(): + parts = [part.strip() for part in line.split(",", maxsplit=2)] + if len(parts) < 2 or not parts[0]: + continue + + name, memory_mib_text = parts[0], parts[1] + # clocks.max.memory is MHz (or "[N/A]" on some cards/drivers). + mem_clock_mhz: float | None = None + if len(parts) == 3: + clock_match = re.search(r"\d+", parts[2]) + if clock_match: + mem_clock_mhz = float(clock_match.group(0)) + + match = re.search(r"\d+", memory_mib_text) + if not match: + if not _is_unified_memory_nvidia_gpu(name): + logger.debug(f"Could not parse nvidia-smi memory value: {line!r}") + continue + gpus.append(_make_nvidia_gpu(name, None, mem_clock_mhz=mem_clock_mhz)) + continue + + memory_mib = int(match.group(0)) + gpus.append( + _make_nvidia_gpu(name, memory_mib * 1024**2, mem_clock_mhz=mem_clock_mhz) + ) + + return gpus + + +def detect_nvidia_gpus() -> list[GPUInfo]: + """Detect NVIDIA GPUs. Returns empty list on failure.""" + try: + import pynvml + except ImportError: + logger.debug("pynvml not installed, trying nvidia-smi fallback") + return _detect_nvidia_gpus_via_smi() + + try: + pynvml.nvmlInit() + except pynvml.NVMLError: + logger.debug("NVML init failed, trying nvidia-smi fallback") + return _detect_nvidia_gpus_via_smi() + + gpus: list[GPUInfo] = [] + try: + count = pynvml.nvmlDeviceGetCount() + # Get CUDA driver version + try: + pynvml.nvmlSystemGetDriverVersion() # ensure driver is accessible + cuda_version = pynvml.nvmlSystemGetCudaDriverVersion_v2() + cuda_str = f"{cuda_version // 1000}.{(cuda_version % 1000) // 10}" + except Exception: + cuda_str = None + + for i in range(count): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + name = pynvml.nvmlDeviceGetName(handle) + if isinstance(name, bytes): + name = name.decode("utf-8") + + try: + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + vram_bytes = mem_info.total + except pynvml.NVMLError: + if not _is_unified_memory_nvidia_gpu(name): + raise + logger.debug(f"NVML did not report dedicated memory for {name}") + vram_bytes = None + + # Max memory clock (MHz) disambiguates same-name memory variants + # (e.g. GTX 1650 GDDR5 vs GDDR6). Optional: not all drivers expose it. + try: + mem_clock_mhz: float | None = float( + pynvml.nvmlDeviceGetMaxClockInfo(handle, pynvml.NVML_CLOCK_MEM) + ) + except Exception as clock_err: + # Optional: without it, same-name memory variants fall back to the + # curated default. Log so an unexpected under-serve is diagnosable. + logger.debug( + f"max mem clock unavailable for {name} " + f"({clock_err}); bandwidth from name only" + ) + mem_clock_mhz = None + + gpus.append(_make_nvidia_gpu(name, vram_bytes, cuda_str, mem_clock_mhz)) + except pynvml.NVMLError as e: + logger.debug(f"Error enumerating NVIDIA GPUs: {e}") + finally: + try: + pynvml.nvmlShutdown() + except Exception: + pass + + if gpus: + return gpus + + logger.debug("NVML returned no NVIDIA GPUs, trying nvidia-smi fallback") + return _detect_nvidia_gpus_via_smi() diff --git a/src/whichllm/hardware/types.py b/src/whichllm/hardware/types.py new file mode 100644 index 0000000..cdca379 --- /dev/null +++ b/src/whichllm/hardware/types.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class GPUInfo: + name: str + vendor: str # "nvidia" | "amd" | "apple" | "intel" + vram_bytes: int + usable_vram_bytes: int | None = None + compute_capability: tuple[int, int] | None = None # NVIDIA only + cuda_version: str | None = None + rocm_version: str | None = None + memory_bandwidth_gbps: float | None = None # from lookup table + shared_memory: bool = False + vram_overridden: bool = False + + +@dataclass +class HardwareInfo: + gpus: list[GPUInfo] = field(default_factory=list) + cpu_name: str = "Unknown" + cpu_cores: int = 0 + has_avx2: bool = False + has_avx512: bool = False + ram_bytes: int = 0 + ram_budget_bytes: int | None = None + disk_free_bytes: int = 0 + os: str = "linux" # "linux" | "darwin" | "windows" + budget_notes: list[str] = field(default_factory=list) diff --git a/src/whichllm/hardware/windows.py b/src/whichllm/hardware/windows.py new file mode 100644 index 0000000..0149b01 --- /dev/null +++ b/src/whichllm/hardware/windows.py @@ -0,0 +1,182 @@ +"""Windows GPU detection via Win32_VideoController.""" + +from __future__ import annotations + +import json +import logging +import re +import subprocess + +from whichllm.constants import AMD_SHARED_MEMORY_APU_MARKERS, _GiB +from whichllm.hardware.gpu_db import resolve_detected_bandwidth +from whichllm.hardware.types import GPUInfo + +logger = logging.getLogger(__name__) + +_WINDOWS_DISCRETE_VRAM_FLOORS: tuple[tuple[str, int], ...] = ( + # AMD sells RX 9060 XT in 8 GB and 16 GB variants. WMI AdapterRAM is a + # uint32 and can report ~4 GB for larger cards, so keep a conservative + # known floor instead of trusting the capped value. + ("RX 9060 XT", 8 * _GiB), +) + + +def _vendor_from_name(name: str) -> str | None: + name_lower = name.lower() + if any(token in name_lower for token in ("amd", "radeon")): + return "amd" + if "intel" in name_lower: + return "intel" + return None + + +def _parse_memory_value(value: object) -> int: + if value is None: + return 0 + try: + ram = int(value) + except (TypeError, ValueError): + return 0 + return max(ram, 0) + + +def _is_amd_shared_memory_apu(name: str) -> bool: + name_upper = name.upper() + return any(marker in name_upper for marker in AMD_SHARED_MEMORY_APU_MARKERS) + + +def _is_intel_discrete_gpu(name: str) -> bool: + return ( + re.search( + r"\barc(?:\(tm\))?\s+(?:pro\s+)?[ab]\d{2,3}", + name, + re.IGNORECASE, + ) + is not None + ) + + +def _is_intel_shared_memory_gpu(name: str, vram_bytes: int) -> bool: + name_lower = name.lower() + if _is_intel_discrete_gpu(name): + return False + if any( + token in name_lower + for token in ( + "uhd", + "iris", + " xe", + "hd graphics", + "arc(tm) graphics", + "intel(r) graphics", + ) + ): + return True + return vram_bytes < 2 * _GiB + + +def _is_shared_memory_gpu(name: str, vendor: str, vram_bytes: int) -> bool: + if vendor == "amd": + return _is_amd_shared_memory_apu(name) + if vendor == "intel": + return _is_intel_shared_memory_gpu(name, vram_bytes) + return False + + +def _apply_discrete_vram_floor(name: str, vram_bytes: int) -> int: + name_upper = name.upper() + for marker, floor in _WINDOWS_DISCRETE_VRAM_FLOORS: + if marker in name_upper and 0 < vram_bytes < floor: + return floor + return vram_bytes + + +def _memory_from_entry(entry: dict) -> int: + dedicated = _parse_memory_value(entry.get("DedicatedVideoMemory")) + if dedicated > 0: + return dedicated + return _parse_memory_value(entry.get("AdapterRAM")) + + +def detect_windows_gpus() -> list[GPUInfo]: + """Detect non-NVIDIA Windows GPUs. Returns empty list on failure.""" + try: + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-Command", + ( + "$controllers = Get-CimInstance Win32_VideoController; " + "$controllers | ForEach-Object { " + "$dedicated = $null; " + "if ($_.PNPDeviceID) { " + "try { " + "$enumPath = 'Registry::HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Enum\\' " + "+ $_.PNPDeviceID + '\\Device Parameters'; " + "$enumProps = Get-ItemProperty -LiteralPath $enumPath -ErrorAction Stop; " + "if ($enumProps.VideoID) { " + "$videoPath = 'Registry::HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Video\\' " + "+ $enumProps.VideoID + '\\0000'; " + "$videoProps = Get-ItemProperty -LiteralPath $videoPath -ErrorAction Stop; " + "$dedicated = $videoProps.'HardwareInformation.qwMemorySize'; " + "} " + "} catch {} " + "} " + "[PSCustomObject]@{" + "Name=$_.Name; " + "AdapterRAM=$_.AdapterRAM; " + "DedicatedVideoMemory=$dedicated" + "} " + "} | ConvertTo-Json -Depth 3" + ), + ], + capture_output=True, + text=True, + timeout=10, + ) + except (FileNotFoundError, subprocess.SubprocessError, OSError) as e: + logger.debug(f"Windows GPU detection failed: {e}") + return [] + + if result.returncode != 0 or not result.stdout.strip(): + return [] + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + logger.debug("Failed to parse Windows GPU JSON") + return [] + + entries = data if isinstance(data, list) else [data] + gpus: list[GPUInfo] = [] + seen: set[str] = set() + for entry in entries: + if not isinstance(entry, dict): + continue + name = str(entry.get("Name") or "").strip() + if not name: + continue + vendor = _vendor_from_name(name) + if vendor is None: + continue + vram_bytes = _memory_from_entry(entry) + shared_memory = _is_shared_memory_gpu(name, vendor, vram_bytes) + if shared_memory: + vram_bytes = 0 + else: + vram_bytes = _apply_discrete_vram_floor(name, vram_bytes) + key = f"{vendor}:{name}:{vram_bytes}" + if key in seen: + continue + seen.add(key) + gpus.append( + GPUInfo( + name=name, + vendor=vendor, + vram_bytes=vram_bytes, + memory_bandwidth_gbps=resolve_detected_bandwidth(name, vram_bytes), + shared_memory=shared_memory, + ) + ) + return gpus diff --git a/src/whichllm/models/__init__.py b/src/whichllm/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/whichllm/models/artifacts.py b/src/whichllm/models/artifacts.py new file mode 100644 index 0000000..d7687bf --- /dev/null +++ b/src/whichllm/models/artifacts.py @@ -0,0 +1,105 @@ +"""Resolve runnable/downloadable model artifacts for ranked recommendations.""" + +from __future__ import annotations + +from whichllm.engine.types import CompatibilityResult +from whichllm.models.types import GGUFVariant, ModelInfo + + +def find_gguf_variant(model: ModelInfo, quant_type: str) -> GGUFVariant | None: + """Return the model's GGUF variant for a quantization type.""" + for variant in model.gguf_variants: + if variant.quant_type.upper() == quant_type.upper(): + return variant + return None + + +def is_same_model_family(candidate: ModelInfo, selected: ModelInfo) -> bool: + """Return whether two repos represent the same base model family.""" + if candidate.id == selected.id: + return True + if candidate.family_id and selected.family_id: + if candidate.family_id == selected.family_id: + return True + if candidate.base_model and candidate.base_model == selected.id: + return True + if selected.base_model and selected.base_model == candidate.id: + return True + if candidate.base_model and selected.base_model: + return candidate.base_model == selected.base_model + return False + + +def has_compatible_parameter_count(candidate: ModelInfo, selected: ModelInfo) -> bool: + """Reject artifact repos that are clearly a different model size.""" + if candidate.parameter_count <= 0 or selected.parameter_count <= 0: + return True + smaller = min(candidate.parameter_count, selected.parameter_count) + larger = max(candidate.parameter_count, selected.parameter_count) + return (larger / smaller) <= 2.0 + + +def resolve_ranked_gguf_artifact( + selected_model: ModelInfo, + selected_variant: GGUFVariant, + models: list[ModelInfo], + quant_filter: str | None = None, +) -> tuple[ModelInfo, GGUFVariant] | None: + """Resolve a ranked GGUF candidate to a real HF repo/file. + + The ranker may synthesize GGUF variants for official safetensors-only repos + so they can be scored realistically. Output surfaces and `run` need the + actual GGUF repository and filename when one exists. + """ + desired_quant = quant_filter or selected_variant.quant_type + + if selected_model.gguf_variants: + variant = find_gguf_variant(selected_model, desired_quant) + return (selected_model, variant) if variant else None + + candidates: list[tuple[bool, int, int, ModelInfo, GGUFVariant]] = [] + for model in models: + if not model.gguf_variants or not is_same_model_family(model, selected_model): + continue + if not has_compatible_parameter_count(model, selected_model): + continue + variant = find_gguf_variant(model, desired_quant) + if not variant: + continue + explicit_base = model.base_model == selected_model.id + candidates.append( + ( + explicit_base, + model.downloads, + model.likes, + model, + variant, + ) + ) + + if not candidates: + return None + + _, _, _, model, variant = max(candidates, key=lambda item: item[:3]) + return model, variant + + +def attach_resolved_artifacts( + results: list[CompatibilityResult], + models: list[ModelInfo], + quant_filter: str | None = None, +) -> None: + """Populate artifact fields on ranked results when a real artifact exists.""" + for result in results: + result.artifact_model = None + result.artifact_variant = None + if not result.gguf_variant: + continue + resolved = resolve_ranked_gguf_artifact( + result.model, + result.gguf_variant, + models, + quant_filter=quant_filter, + ) + if resolved: + result.artifact_model, result.artifact_variant = resolved diff --git a/src/whichllm/models/benchmark.py b/src/whichllm/models/benchmark.py new file mode 100644 index 0000000..15b65bb --- /dev/null +++ b/src/whichllm/models/benchmark.py @@ -0,0 +1,90 @@ +"""Compatibility shim for benchmark fetching and lookup helpers. + +The implementation is split by responsibility: + +- ``benchmark_cache`` for cache I/O +- ``benchmark_fetch`` for source fetching and layered merge policy +- ``benchmark_lineage`` for frozen-score recency demotion +- ``benchmark_index`` for score indices and model-line interpolation +- ``benchmark_lookup`` for evidence lookup and inheritance rules + +Existing imports from ``whichllm.models.benchmark`` are re-exported here. +""" + +from __future__ import annotations + +from whichllm.models import benchmark_cache as _cache_module +from whichllm.models.benchmark_cache import DEFAULT_TTL_SECONDS +from whichllm.models.benchmark_fetch import fetch_benchmark_scores +from whichllm.models.benchmark_index import ( + _extract_model_lines, + _extract_params_b_from_id, + _interpolate_line_score, + build_line_bucket_index, + build_score_index, +) +from whichllm.models.benchmark_lineage import ( + _apply_lineage_recency_demotion, + _build_lineage_regex, + _lineage_recency_factor, +) +from whichllm.models.benchmark_lookup import ( + _REPO_SUFFIXES, + _append_unique, + _generate_candidates, + _generate_score_name_candidates, + _params_compatible, + _strip_repo_suffix, + _try_lookup, + lookup_benchmark, + lookup_benchmark_evidence, +) +from whichllm.models.benchmark_types import BenchmarkEvidence + +CACHE_DIR = _cache_module.CACHE_DIR +BENCHMARK_CACHE = _cache_module.BENCHMARK_CACHE + + +def _sync_cache_module_globals() -> None: + _cache_module.CACHE_DIR = CACHE_DIR + _cache_module.BENCHMARK_CACHE = BENCHMARK_CACHE + + +def load_benchmark_cache() -> dict[str, float] | None: + """Load cached benchmark scores through the legacy shim globals.""" + _sync_cache_module_globals() + return _cache_module.load_benchmark_cache() + + +def save_benchmark_cache(scores: dict[str, float]) -> None: + """Save cached benchmark scores through the legacy shim globals.""" + _sync_cache_module_globals() + _cache_module.save_benchmark_cache(scores) + + +__all__ = [ + "BENCHMARK_CACHE", + "CACHE_DIR", + "DEFAULT_TTL_SECONDS", + "BenchmarkEvidence", + "_REPO_SUFFIXES", + "_append_unique", + "_apply_lineage_recency_demotion", + "_build_lineage_regex", + "_extract_model_lines", + "_extract_params_b_from_id", + "_generate_candidates", + "_generate_score_name_candidates", + "_interpolate_line_score", + "_lineage_recency_factor", + "_params_compatible", + "_strip_repo_suffix", + "_try_lookup", + "build_line_bucket_index", + "build_score_index", + "fetch_benchmark_scores", + "load_benchmark_cache", + "lookup_benchmark", + "lookup_benchmark_evidence", + "save_benchmark_cache", +] diff --git a/src/whichllm/models/benchmark_cache.py b/src/whichllm/models/benchmark_cache.py new file mode 100644 index 0000000..8c2bfec --- /dev/null +++ b/src/whichllm/models/benchmark_cache.py @@ -0,0 +1,39 @@ +"""Benchmark score cache helpers.""" + +from __future__ import annotations + +import json +import logging +import time + +from whichllm.utils import _cache_dir + +logger = logging.getLogger(__name__) + +CACHE_DIR = _cache_dir() +BENCHMARK_CACHE = CACHE_DIR / "benchmark.json" +DEFAULT_TTL_SECONDS = 24 * 3600 # 24 hours + + +def load_benchmark_cache() -> dict[str, float] | None: + """Load cached benchmark scores. Returns None if expired or missing.""" + if not BENCHMARK_CACHE.exists(): + return None + try: + data = json.loads(BENCHMARK_CACHE.read_text(encoding="utf-8")) + cached_at = data.get("cached_at", 0) + if time.time() - cached_at > DEFAULT_TTL_SECONDS: + logger.debug("Benchmark cache expired") + return None + return data.get("scores", {}) + except (json.JSONDecodeError, KeyError) as e: + logger.debug(f"Benchmark cache corrupted: {e}") + return None + + +def save_benchmark_cache(scores: dict[str, float]) -> None: + """Save benchmark scores to cache.""" + CACHE_DIR.mkdir(parents=True, exist_ok=True) + data = {"cached_at": time.time(), "scores": scores} + BENCHMARK_CACHE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + logger.debug(f"Saved {len(scores)} benchmark scores to cache") diff --git a/src/whichllm/models/benchmark_fetch.py b/src/whichllm/models/benchmark_fetch.py new file mode 100644 index 0000000..65cede6 --- /dev/null +++ b/src/whichllm/models/benchmark_fetch.py @@ -0,0 +1,119 @@ +"""Fetch and merge benchmark scores from source adapters.""" + +from __future__ import annotations + +import asyncio +import logging + +import httpx + +from whichllm.models.benchmark_lineage import _apply_lineage_recency_demotion +from whichllm.models.http import DEFAULT_ACCEPT_ENCODING +from whichllm.utils import _current_version + +logger = logging.getLogger(__name__) + + +async def fetch_benchmark_scores() -> dict[str, float]: + """Fetch and combine benchmark scores from multiple sources. + + Sources, merged in this order (later overwrites earlier on conflict): + 1. Open LLM Leaderboard v2 (archived 2025-06) + 2. Chatbot Arena ELO (frozen 2025-07-17) + 3. LiveBench (vendored snapshot) + 4. Aider polyglot (coding-specific) + 5. Artificial Analysis Intelligence Index + 6. Vision-language capability index + + Returns dict mapping model_id -> normalized score (0-100). All network + sources are fetched concurrently; failures are logged and skipped. + """ + from whichllm.models import benchmark_sources + + async with httpx.AsyncClient( + timeout=30.0, + follow_redirects=True, + headers={ + "Accept-Encoding": DEFAULT_ACCEPT_ENCODING, + "User-Agent": f"whichllm/{_current_version()}", + }, + ) as client: + leaderboard_task = asyncio.create_task( + benchmark_sources.fetch_leaderboard_with_fallback(client) + ) + arena_task = asyncio.create_task(benchmark_sources.fetch_arena_scores(client)) + aa_task = asyncio.create_task(benchmark_sources.fetch_aa_index_scores(client)) + aider_task = asyncio.create_task( + benchmark_sources.fetch_aider_polyglot_scores(client) + ) + vision_task = asyncio.create_task(benchmark_sources.fetch_vision_scores(client)) + + ( + lb_result, + arena_result, + aa_result, + aider_result, + vision_result, + ) = await asyncio.gather( + leaderboard_task, + arena_task, + aa_task, + aider_task, + vision_task, + return_exceptions=True, + ) + + frozen: dict[str, float] = {} + current: dict[str, float] = {} + + if isinstance(lb_result, BaseException): + logger.warning(f"Leaderboard fetch failed: {lb_result}") + else: + frozen.update(lb_result) + logger.debug(f"Leaderboard: {len(lb_result)} scores (frozen)") + + if isinstance(arena_result, BaseException): + logger.warning(f"Arena fetch failed, using fallback: {arena_result}") + else: + for k, v in arena_result.items(): + if frozen.get(k, 0.0) < v: + frozen[k] = v + logger.debug(f"Arena: {len(arena_result)} scores (frozen)") + + livebench_result = benchmark_sources.get_livebench_data() + for k, v in livebench_result.items(): + if current.get(k, 0.0) < v: + current[k] = v + logger.debug(f"LiveBench: {len(livebench_result)} scores (current)") + + if isinstance(aa_result, BaseException): + logger.warning(f"AA Index fetch failed, will use fallback: {aa_result}") + aa_result = benchmark_sources.get_aa_curated_fallback() + + for k, v in aa_result.items(): + if current.get(k, 0.0) < v: + current[k] = v + logger.debug(f"AA Index: {len(aa_result)} scores (current)") + + if isinstance(aider_result, BaseException): + logger.warning(f"Aider fetch failed: {aider_result}") + else: + for k, v in aider_result.items(): + if current.get(k, 0.0) < v * 0.85: + current[k] = v * 0.85 + logger.debug(f"Aider polyglot: {len(aider_result)} scores (current, 0.85x)") + + if isinstance(vision_result, BaseException): + logger.warning(f"Vision fetch failed: {vision_result}") + else: + for k, v in vision_result.items(): + if current.get(k, 0.0) < v: + current[k] = v + logger.debug(f"Vision: {len(vision_result)} scores (current)") + + combined: dict[str, float] = dict(frozen) + combined.update(current) + combined = _apply_lineage_recency_demotion(combined, frozen, current) + + logger.debug(f"Combined: {len(combined)} benchmark scores") + return combined diff --git a/src/whichllm/models/benchmark_index.py b/src/whichllm/models/benchmark_index.py new file mode 100644 index 0000000..08049f5 --- /dev/null +++ b/src/whichllm/models/benchmark_index.py @@ -0,0 +1,117 @@ +"""Benchmark score indexing and model-line interpolation.""" + +from __future__ import annotations + +import math +import re +import statistics + + +def _extract_params_b_from_id(model_id: str) -> float | None: + """Extract parameter size in billions from model ID text.""" + lower = model_id.lower() + matches = re.findall(r"(\d+(?:\.\d+)?)b(?:-a\d+(?:\.\d+)?b)?", lower) + if not matches: + return None + try: + return max(float(v) for v in matches) + except ValueError: + return None + + +def _extract_model_lines(model_id: str) -> list[str]: + """Extract model line candidates from a model ID, most specific first.""" + if "/" not in model_id: + return [] + lower = model_id.lower() + + stripped = re.sub(r"-(gguf|awq|gptq|fp8|fp16|bf16|mxfp4|nvfp4)$", "", lower) + stripped = re.sub(r"-\d{4}(-hf)?$", "", stripped) + + lines: list[str] = [] + cleaned = re.sub( + r"-\d+(\.\d+)?b(-a\d+b)?(-[a-z][-a-z0-9]*)*$", + "", + stripped, + ) + if cleaned != stripped and "/" in cleaned: + lines.append(cleaned) + + for line in list(lines) + ([stripped] if not lines else []): + broader = re.sub(r"(\d+)\.\d+$", r"\1", line) + if broader != line and broader not in lines: + lines.append(broader) + + return lines + + +def _interpolate_line_score( + bucket: list[tuple[float | None, float]], + params_b: float | None, +) -> tuple[float, float]: + """Interpolate score from same-model-line benchmarks with confidence.""" + if not bucket: + return 0.0, 0.0 + + valid = [(p, s) for p, s in bucket if p is not None] + if not valid: + vals = [s for _, s in bucket] + return statistics.median(vals), 0.25 + + if params_b is None or params_b <= 0: + vals = [s for _, s in valid] + return statistics.median(vals), 0.30 + + weighted: list[tuple[float, float, float]] = [] + for p, s in valid: + assert p is not None + dist = abs(math.log2(max(params_b, 0.1) / max(p, 0.1))) + w = 1.0 / (0.35 + dist) + weighted.append((w, s, dist)) + + score = sum(w * s for w, s, _ in weighted) / sum(w for w, _, _ in weighted) + nearest = min(d for _, _, d in weighted) + if nearest <= 0.15: + conf = 0.45 + elif nearest <= 0.50: + conf = 0.34 + else: + conf = 0.26 + return score, conf + + +def build_score_index( + scores: dict[str, float], +) -> tuple[dict[str, float], dict[str, float]]: + """Build case-insensitive and model-line lookup indices.""" + ci_index: dict[str, float] = {} + line_index: dict[str, float] = {} + + for key, val in scores.items(): + lk = key.lower() + if lk not in ci_index or val > ci_index[lk]: + ci_index[lk] = val + + lines = _extract_model_lines(key) + if not lines and "/" in key: + lines = [lk] + for line in lines: + if line not in line_index or val > line_index[line]: + line_index[line] = val + + return ci_index, line_index + + +def build_line_bucket_index( + scores: dict[str, float], +) -> dict[str, list[tuple[float | None, float]]]: + """Build line -> [(params_b, score)] index for size-aware interpolation.""" + buckets: dict[str, list[tuple[float | None, float]]] = {} + for key, val in scores.items(): + params_b = _extract_params_b_from_id(key) + lines = _extract_model_lines(key) + if not lines and "/" in key: + lines = [key.lower()] + for line in lines: + buckets.setdefault(line, []).append((params_b, val)) + return buckets diff --git a/src/whichllm/models/benchmark_lineage.py b/src/whichllm/models/benchmark_lineage.py new file mode 100644 index 0000000..50cd357 --- /dev/null +++ b/src/whichllm/models/benchmark_lineage.py @@ -0,0 +1,63 @@ +"""Lineage-aware recency demotion for frozen benchmark scores.""" + +from __future__ import annotations + +import re + +_LINEAGE_DEMOTION_REGEX = None + + +def _build_lineage_regex(): + """Compile MODEL_LINEAGE_VERSIONS once into (family, [(re, idx)]) form.""" + global _LINEAGE_DEMOTION_REGEX + if _LINEAGE_DEMOTION_REGEX is not None: + return _LINEAGE_DEMOTION_REGEX + from whichllm.constants import MODEL_LINEAGE_VERSIONS + + out = {} + for family, entries in MODEL_LINEAGE_VERSIONS.items(): + compiled = [(re.compile(pat), idx) for pat, idx in entries] + max_idx = max(idx for _, idx in entries) + out[family] = (compiled, max_idx) + _LINEAGE_DEMOTION_REGEX = out + return out + + +def _lineage_recency_factor(model_id: str) -> float: + """Return a multiplicative recency factor for frozen-only scores. + + Newest generation in a known family -> 1.0 (no demotion). Each generation + older -> another 12% off. Unknown families -> 1.0. + """ + if not model_id: + return 1.0 + lower = model_id.lower() + families = _build_lineage_regex() + best_factor = 1.0 + for family, (patterns, max_idx) in families.items(): + for regex, idx in patterns: + if regex.search(lower): + gens_old = max(0, max_idx - idx) + factor = max(0.55, 1.0 - 0.12 * gens_old) + if factor < best_factor: + best_factor = factor + break + return best_factor + + +def _apply_lineage_recency_demotion( + combined: dict[str, float], + frozen: dict[str, float], + current: dict[str, float], +) -> dict[str, float]: + """Multiply frozen-only entries by a lineage-derived recency factor.""" + if not combined: + return combined + out: dict[str, float] = {} + for k, v in combined.items(): + if k in current: + out[k] = v + continue + factor = _lineage_recency_factor(k) + out[k] = round(v * factor, 1) + return out diff --git a/src/whichllm/models/benchmark_lookup.py b/src/whichllm/models/benchmark_lookup.py new file mode 100644 index 0000000..e9e52a8 --- /dev/null +++ b/src/whichllm/models/benchmark_lookup.py @@ -0,0 +1,192 @@ +"""Benchmark evidence lookup and inheritance rules.""" + +from __future__ import annotations + +from whichllm.models.benchmark_index import ( + _extract_model_lines, + _extract_params_b_from_id, + _interpolate_line_score, + build_line_bucket_index, + build_score_index, +) +from whichllm.models.benchmark_types import BenchmarkEvidence + + +def _try_lookup( + candidate: str, scores: dict[str, float], ci_index: dict[str, float] +) -> float | None: + """Try exact match, then case-insensitive match.""" + if candidate in scores: + return scores[candidate] + lc = candidate.lower() + if lc in ci_index: + return ci_index[lc] + return None + + +_REPO_SUFFIXES = ("-GGUF", "-gguf", "-AWQ", "-GPTQ", "-FP8", "-fp8", "-BF16", "-bf16") + + +def _generate_candidates(model_id: str) -> list[str]: + """Generate candidate IDs to look up for a model.""" + candidates = [model_id] + + for suffix in _REPO_SUFFIXES: + if model_id.endswith(suffix): + candidates.append(model_id[: -len(suffix)]) + break + + base = candidates[-1] + if base.endswith("-Instruct"): + candidates.append(base[: -len("-Instruct")]) + else: + candidates.append(base + "-Instruct") + + return candidates + + +def _append_unique(candidates: list[str], candidate: str) -> None: + if candidate and candidate not in candidates: + candidates.append(candidate) + + +def _strip_repo_suffix(model_id: str) -> str: + for suffix in _REPO_SUFFIXES: + if model_id.endswith(suffix): + return model_id[: -len(suffix)] + return model_id + + +def _generate_score_name_candidates( + model_id: str, scores: dict[str, float] +) -> list[str]: + """Match community repo names to benchmark IDs with the same model name.""" + stripped = _strip_repo_suffix(model_id) + repo_name = stripped.rsplit("/", 1)[-1] + model_names = [repo_name] + + explicit_candidates: list[str] = [] + if "_" in repo_name: + org, name = repo_name.split("_", 1) + if org and name: + _append_unique(explicit_candidates, f"{org}/{name}") + _append_unique(model_names, name) + + score_candidates: list[str] = [] + wanted_names = {name.lower() for name in model_names if name} + for score_id in scores: + score_name = score_id.rsplit("/", 1)[-1].lower() + if score_name in wanted_names: + _append_unique(score_candidates, score_id) + + return explicit_candidates + [ + candidate + for candidate in score_candidates + if candidate not in explicit_candidates + ] + + +def lookup_benchmark( + model_id: str, + base_model: str | None, + scores: dict[str, float], + ci_index: dict[str, float] | None = None, + line_index: dict[str, float] | None = None, +) -> tuple[float, bool] | None: + """Backward-compatible benchmark lookup helper.""" + evidence = lookup_benchmark_evidence( + model_id, + base_model, + scores, + ci_index=ci_index, + line_index=line_index, + ) + if evidence.score is None: + return None + return evidence.score, evidence.source == "direct" + + +def _params_compatible(actual_b: float | None, ref_id: str) -> bool: + """Reject benchmark inheritance when actual and reference sizes diverge.""" + if actual_b is None or actual_b <= 0: + return True + ref_b = _extract_params_b_from_id(ref_id) + if ref_b is None or ref_b <= 0: + return True + ratio = actual_b / ref_b + return 0.5 <= ratio <= 2.0 + + +def lookup_benchmark_evidence( + model_id: str, + base_model: str | None, + scores: dict[str, float], + ci_index: dict[str, float] | None = None, + line_index: dict[str, float] | None = None, + line_bucket_index: dict[str, list[tuple[float | None, float]]] | None = None, + self_reported_score: float | None = None, + actual_params_b: float | None = None, +) -> BenchmarkEvidence: + """Look up benchmark evidence with confidence.""" + if ci_index is None or line_index is None: + ci_index, line_index = build_score_index(scores) + if line_bucket_index is None: + line_bucket_index = build_line_bucket_index(scores) + + direct_result = _try_lookup(model_id, scores, ci_index) + if direct_result is not None: + return BenchmarkEvidence(score=direct_result, confidence=1.0, source="direct") + + variant_candidates = _generate_candidates(model_id)[1:] + for candidate in _generate_score_name_candidates(model_id, scores): + _append_unique(variant_candidates, candidate) + for candidate in variant_candidates: + result = _try_lookup(candidate, scores, ci_index) + if result is not None: + if not _params_compatible(actual_params_b, candidate): + continue + return BenchmarkEvidence(score=result, confidence=0.55, source="variant") + + if base_model: + for candidate in _generate_candidates(base_model): + result = _try_lookup(candidate, scores, ci_index) + if result is not None: + if not _params_compatible(actual_params_b, candidate): + continue + return BenchmarkEvidence( + score=result, confidence=0.60, source="base_model" + ) + + size_hint = ( + actual_params_b + or _extract_params_b_from_id(model_id) + or _extract_params_b_from_id(base_model or "") + ) + for mid in (model_id, base_model): + if mid: + for line in _extract_model_lines(mid): + if line in line_bucket_index: + score, conf = _interpolate_line_score( + line_bucket_index[line], size_hint + ) + if score > 0: + return BenchmarkEvidence( + score=score, confidence=conf, source="line_interp" + ) + if line in line_index: + return BenchmarkEvidence( + score=line_index[line], confidence=0.22, source="line_interp" + ) + + if ( + self_reported_score is not None + and isinstance(self_reported_score, (int, float)) + and self_reported_score > 0 + ): + return BenchmarkEvidence( + score=float(self_reported_score), + confidence=0.40, + source="self_reported", + ) + + return BenchmarkEvidence(score=None, confidence=0.0, source="none") diff --git a/src/whichllm/models/benchmark_sources/__init__.py b/src/whichllm/models/benchmark_sources/__init__.py new file mode 100644 index 0000000..a54a21b --- /dev/null +++ b/src/whichllm/models/benchmark_sources/__init__.py @@ -0,0 +1,43 @@ +"""External benchmark sources beyond Chatbot Arena and Open LLM Leaderboard. + +Each module here fetches an independent leaderboard / index, normalizes it to +the same 0-100 scale, and returns a ``dict[str, float]`` keyed by HuggingFace +model id (or a list of synonyms). + +The functions are intentionally defensive: if a source is unreachable or +returns malformed data, they log a warning and return an empty dict so the +main benchmark merge pipeline does not abort. +""" + +from whichllm.models.benchmark_sources.aa_index import ( + fetch_aa_index_scores, + get_aa_curated_fallback, +) +from whichllm.models.benchmark_sources.aider import fetch_aider_polyglot_scores +from whichllm.models.benchmark_sources.chatbot_arena import fetch_arena_scores +from whichllm.models.benchmark_sources.livebench import ( + get_livebench_data, +) +from whichllm.models.benchmark_sources.open_llm_leaderboard import ( + fetch_leaderboard_with_fallback, +) +from whichllm.models.benchmark_sources.vision import fetch_vision_scores + +# Newest curated-fallback date across all sources. Live scrapes are merged +# on top when reachable, but they frequently are not (the leaderboard +# spaces change their JSON shape), so the user-visible ranking is anchored +# to this snapshot. Surface it in the CLI so a stale recommendation is +# self-evident rather than silently trusted. Bump this whenever any +# *_FALLBACK_* dict is refreshed. +BENCHMARK_SNAPSHOT = "2026-05" + +__all__ = [ + "BENCHMARK_SNAPSHOT", + "fetch_aa_index_scores", + "fetch_aider_polyglot_scores", + "fetch_arena_scores", + "fetch_leaderboard_with_fallback", + "fetch_vision_scores", + "get_aa_curated_fallback", + "get_livebench_data", +] diff --git a/src/whichllm/models/benchmark_sources/aa_index.py b/src/whichllm/models/benchmark_sources/aa_index.py new file mode 100644 index 0000000..77fcdea --- /dev/null +++ b/src/whichllm/models/benchmark_sources/aa_index.py @@ -0,0 +1,386 @@ +"""Artificial Analysis Intelligence Index source. + +AA publishes a model-quality index (https://artificialanalysis.ai/) that +covers post-2025-08 frontier releases (DeepSeek V4, GLM-5, Kimi K2.6, +MiMo V2.5, Qwen3.6, etc.) that whichllm's primary sources have stopped +tracking. The index is exposed via the JSON payload embedded in +``__NEXT_DATA__`` on the leaderboard page. + +The fetcher is defensive: any failure (network, schema drift, parsing) is +caught and an empty dict is returned so it never blocks the main benchmark +pipeline. +""" + +from __future__ import annotations + +import json +import logging +import re + +import httpx + +from whichllm.models.benchmark_sources.constants import _NEXT_DATA_RE +from whichllm.models.benchmark_sources.types import ExtractionFailed +from whichllm.models.benchmark_sources.utils import _walk +from whichllm.models.http import get_with_retries + +logger = logging.getLogger(__name__) + +# Display name -> list of (org_prefix, repo_name_candidates) tuples used to +# map AA-reported labels back to HuggingFace model IDs. Only the most common +# fully-open-weights releases need entries here; anything else is dropped. +AA_NAME_TO_HF_IDS: dict[str, list[str]] = { + "Kimi K2": ["moonshotai/Kimi-K2-Instruct", "moonshotai/Kimi-K2-Base"], + "Kimi K2-Thinking": ["moonshotai/Kimi-K2-Thinking"], + "DeepSeek V3": ["deepseek-ai/DeepSeek-V3", "deepseek-ai/DeepSeek-V3-0324"], + "DeepSeek V3.1": ["deepseek-ai/DeepSeek-V3.1"], + "DeepSeek V3.2": ["deepseek-ai/DeepSeek-V3.2"], + "DeepSeek V3.2-Exp": ["deepseek-ai/DeepSeek-V3.2-Exp"], + "DeepSeek V4 Pro": ["deepseek-ai/DeepSeek-V4-Pro"], + "DeepSeek V4 Flash": ["deepseek-ai/DeepSeek-V4-Flash"], + "DeepSeek R1": ["deepseek-ai/DeepSeek-R1"], + "DeepSeek R1-0528": ["deepseek-ai/DeepSeek-R1-0528"], + "DeepSeek R1-Distill 32B": ["deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"], + "DeepSeek R1-Distill 14B": ["deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"], + "DeepSeek R1-Distill 8B": ["deepseek-ai/DeepSeek-R1-Distill-Llama-8B"], + "QwQ 32B": ["Qwen/QwQ-32B"], + "Qwen3 4B Thinking": ["Qwen/Qwen3-4B-Thinking-2507"], + "MiMo V2.5": ["XiaomiMiMo/MiMo-V2.5"], + "MiMo V2.5 Pro": ["XiaomiMiMo/MiMo-V2.5-Pro"], + "MiMo V2 Flash": ["XiaomiMiMo/MiMo-V2-Flash"], + "GLM-4.5": ["zai-org/GLM-4.5", "zai-org/GLM-4.5-Air"], + "GLM-4.6": ["zai-org/GLM-4.6"], + "GLM-4.7": ["zai-org/GLM-4.7"], + "GLM-4.7-Flash": ["zai-org/GLM-4.7-Flash"], + "GLM-5": ["zai-org/GLM-5", "zai-org/GLM-5-FP8"], + "GLM-5.1": ["zai-org/GLM-5.1", "zai-org/GLM-5.1-FP8"], + "gpt-oss-20b": ["openai/gpt-oss-20b"], + "gpt-oss-120b": ["openai/gpt-oss-120b"], + "Qwen3-Next 80B-A3B": ["Qwen/Qwen3-Next-80B-A3B-Instruct"], + "Qwen3.5 397B-A17B": ["Qwen/Qwen3.5-397B-A17B"], + "Qwen3 235B-A22B": ["Qwen/Qwen3-235B-A22B"], + "Qwen3 32B": ["Qwen/Qwen3-32B"], + "Qwen3 14B": ["Qwen/Qwen3-14B"], + "Qwen3 8B": ["Qwen/Qwen3-8B"], + "Qwen3-VL 235B-A22B": ["Qwen/Qwen3-VL-235B-A22B-Instruct"], + "Llama 3.3 70B": ["meta-llama/Llama-3.3-70B-Instruct"], + "Llama 4 Scout": ["meta-llama/Llama-4-Scout-17B-16E-Instruct"], + "Llama 4 Maverick": ["meta-llama/Llama-4-Maverick-17B-128E-Instruct"], + "Gemma 3 27B": ["google/gemma-3-27b-it"], + "Gemma 3 12B": ["google/gemma-3-12b-it"], + "Gemma 4 31B": ["google/gemma-4-31b-it"], + "Gemma 4 26B-A4B": ["google/gemma-4-26b-a4b-it"], + "Mistral Large 2": ["mistralai/Mistral-Large-Instruct-2411"], + "Devstral Small": ["mistralai/Devstral-Small-2505"], + "Phi-4": ["microsoft/phi-4"], + "Command A": ["CohereForAI/c4ai-command-a-03-2025"], + "Command R+": [ + "CohereForAI/c4ai-command-r-plus-08-2024", + "CohereForAI/c4ai-command-r-plus", + ], + "MiniMax-M2": ["MiniMaxAI/MiniMax-M2"], + "MiniMax-M2.5": ["MiniMaxAI/MiniMax-M2.5"], + "Nemotron 3 Super 120B-A12B": ["nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"], + "Nemotron 3 Nano 30B-A3B": [ + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", + ], +} + +# Bounds used to normalize raw AA index values onto the 0-100 scale the rest +# of the ranking system uses. AA reworked their Intelligence Index in +# 2026-Q2 and the open-weights distribution compressed sharply (top open model +# ~44, 8B-class ~7, weakest mapped repos ~3). The window is anchored by the +# same two-point fit as before: top open frontier (DeepSeek-V4-Pro = 44.3) → 95 +# normalized, and 8B-class (Qwen3-8B = 7.4) → 40 normalized. This keeps a strong +# 8B model competitive with frozen-OLLB 7B scores while leaving headroom for +# frontier-tier models. On the reworked scale that fit puts the floor below +# zero; that is fine, live AA values are always positive and clamp at 0. +_AA_INDEX_MIN = -19.4 +_AA_INDEX_MAX = 47.6 + +AA_LEADERBOARD_URL = "https://artificialanalysis.ai/leaderboards/models" + +# Snapshot of the AA Intelligence Index (open-weights only), refreshed on +# 2026-06-29 from artificialanalysis.ai against their reworked index. Used as a +# fallback when the live HTML scrape returns no results (e.g. because the +# Next.js payload format changes again). Entries are raw AA index values, +# normalized through _normalize_aa_index() in get_aa_curated_fallback(). +# +# Entries marked "live" are real values from the reworked index. Entries marked +# "peer" are models AA does not track; their raw value is set so that, under the +# current bounds, they reproduce their previous normalized score (hand-estimated +# LB-equivalents). On the reworked scale several peers fall below the index +# floor and read as negative raw; that is an artifact of reusing the AA-index +# field for non-AA estimates. _normalize_aa_index() clamps them at 0, so the +# normalized output stays correct. (We could instead store this snapshot as +# already-normalized 0-100 values, which would drop the negatives and decouple +# the fallback from future bound retunes, but that is a larger change than this +# issue calls for and is left for a follow-up if you want it.) +AA_INDEX_FALLBACK_2026_06_29: dict[str, float] = { + # Frontier MoE / very large + "moonshotai/Kimi-K2-Thinking": 32.7, # live + "moonshotai/Kimi-K2-Instruct": 19.4, # live + "XiaomiMiMo/MiMo-V2.5-Pro": 42.2, # live + "XiaomiMiMo/MiMo-V2.5": 40.1, # live + "deepseek-ai/DeepSeek-V4-Pro": 44.3, # live + "deepseek-ai/DeepSeek-V4-Flash": 40.3, # live + "deepseek-ai/DeepSeek-V3.2": 33.4, # live + "deepseek-ai/DeepSeek-V3.2-Exp": 25.4, # live + "deepseek-ai/DeepSeek-V3.1": 21.0, # live + "deepseek-ai/DeepSeek-V3-0324": 10.4, # live + "deepseek-ai/DeepSeek-V3": 10.4, # live + "deepseek-ai/DeepSeek-R1-0528": 20.1, # live + "deepseek-ai/DeepSeek-R1": 12.6, # live + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": 10.5, # peer + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": 1.3, # peer + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B": -7.9, # peer + "Qwen/QwQ-32B": 13.4, # live + "Qwen/Qwen3-4B-Thinking-2507": -4.8, # peer + "zai-org/GLM-5.1": 40.2, # live + "zai-org/GLM-5": 39.5, # live + "zai-org/GLM-5-FP8": 39.5, # live + "zai-org/GLM-5.1-FP8": 40.2, # live + "zai-org/GLM-4.7-Flash": 22.9, # live + "zai-org/GLM-4.6": 25.1, # live + "zai-org/GLM-4.5": 19.5, # live + "zai-org/GLM-4.5-Air": 19.5, # live + # Qwen family + "Qwen/Qwen3.6-27B": 32.0, # peer + "Qwen/Qwen3.5-397B-A17B": 33.7, # live + "Qwen/Qwen3-Next-80B-A3B-Instruct": 19.8, # live + "Qwen/Qwen3-235B-A22B": 13.4, # live + "Qwen/Qwen3-Coder-30B-A3B-Instruct": 19.7, # peer + "Qwen/Qwen3-32B": 11.5, # live + "Qwen/Qwen3-14B": 10.1, # live + "Qwen/Qwen3-8B": 7.4, # live + "Qwen/Qwen3-4B-Instruct-2507": 4.4, # peer + "Qwen/Qwen3-4B": 1.3, # peer + "Qwen/Qwen3-1.7B": -7.9, # peer + "Qwen/Qwen3-0.6B": -14.0, # peer + # 8B-class peers (no AA tracking but realistic LB-equivalents) + "meta-llama/Llama-3.1-8B-Instruct": -4.8, # peer + "meta-llama/Meta-Llama-3-8B-Instruct": -7.9, # peer + "google/gemma-2-9b-it": -3.3, # peer + "microsoft/Phi-4-mini-instruct": -1.8, # peer + "mistralai/Mistral-7B-Instruct-v0.3": -7.9, # peer + "Qwen/Qwen2.5-7B-Instruct": -4.8, # peer + "Qwen/Qwen2.5-14B-Instruct": 1.3, # peer + "Qwen/Qwen2.5-32B-Instruct": 7.4, # peer + "Qwen/Qwen3-30B-A3B": 10.5, # peer + # Other major open releases + "openai/gpt-oss-120b": 23.8, # live + "openai/gpt-oss-20b": 14.9, # live + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 14.3, # live + "meta-llama/Llama-4-Scout-17B-16E-Instruct": 10.0, # live + "meta-llama/Llama-3.3-70B-Instruct": 12.0, # peer + "google/gemma-4-31b-it": 29.4, # live + "google/gemma-4-26b-a4b-it": 25.7, # live + "google/gemma-3-27b-it": 12.0, # peer + "google/gemma-3-12b-it": 7.4, # peer + "microsoft/phi-4": 4.9, # live + "mistralai/Mistral-Large-Instruct-2411": 9.1, # live + "mistralai/Mistral-Small-3.2-24B-Instruct-2506": 10.5, # peer + "mistralai/Mistral-Small-3.1-24B-Instruct-2503": 7.4, # peer + "mistralai/Devstral-Small-2505": 11.8, # live + "MiniMaxAI/MiniMax-M2.5": 33.7, # live + "stepfun-ai/Step-3.5-Flash": 19.7, # peer + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": 16.6, # peer + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": 12.0, # peer + # Correct IDs for OLMo / Granite / Codestral families (the earlier + # forecast IDs like "OLMo-3-32B-Instruct" or "granite-4.1-30b-instruct" + # never shipped publicly under those names). + "allenai/Olmo-3-7B-Instruct": -4.8, # peer + "allenai/Olmo-3-1025-7B": -4.8, # peer + "ibm-granite/granite-4.0-h-small": 7.4, # peer + "ibm-granite/granite-4.0-h-tiny": -4.8, # peer + "ibm-granite/granite-3.3-8b-instruct": -3.3, # peer + "ibm-granite/granite-3.3-2b-instruct": -12.5, # peer + "mistralai/Codestral-22B-v0.1": 4.4, # peer +} + + +def _normalize_aa_index(index: float) -> float: + """Normalize a raw AA index value onto the 0-100 scale.""" + if not isinstance(index, (int, float)): + return 0.0 + span = _AA_INDEX_MAX - _AA_INDEX_MIN + normalized = (index - _AA_INDEX_MIN) / span * 100.0 + return max(0.0, min(100.0, round(normalized, 1))) + + +# --- Next.js App Router (RSC) scraping ------------------------------------- +# +# artificialanalysis.ai migrated off the classic ``__NEXT_DATA__`` blob to the +# App Router streaming format: model data arrives in ``self.__next_f.push([n, +# "…"])`` calls whose second element is a JSON-string-escaped fragment of the +# RSC payload. We concatenate + unescape those chunks, then pull every +# ``{"name": …, …, "intelligenceIndex": …}`` record out with a bounded regex +# (the payload is a flat RSC stream, not a single parseable JSON document). + +_RSC_CHUNK_RE = re.compile(r'self\.__next_f\.push\(\[\d+,(?P"(?:[^"\\]|\\.)*")\]\)') +# A model record: a "name" string followed, within the SAME object, by its +# "intelligenceIndex". The middle group forbids another '"name":"' so the +# match cannot leak across into a neighbouring record that lacks an index. +_AA_RECORD_RE = re.compile( + r'"name":"(?P(?:[^"\\]|\\.)*)"' + r'(?:(?!"name":").)*?' + r'"intelligenceIndex":(?P-?\d+(?:\.\d+)?)', + re.DOTALL, +) +# Variant qualifiers AA appends that don't change the underlying HF weights: +# "(Reasoning)", "(Non-reasoning)", "(high)", "(Reasoning, Max Effort)", etc. +_PAREN_RE = re.compile(r"\([^)]*\)") + + +def _canonical_name(name: str) -> str: + """Normalize an AA display name for fuzzy matching against the HF map. + + Drops parenthetical variant qualifiers and collapses separator/case noise + so ``"Qwen3 14B (Reasoning)"`` and ``"Qwen3-14B"`` both canonicalize to + ``"qwen3 14b"``. + """ + name = _PAREN_RE.sub("", name) + name = name.lower().replace("-", " ").replace("_", " ") + return re.sub(r"\s+", " ", name).strip() + + +# Canonical-name -> HF ids, derived once from AA_NAME_TO_HF_IDS. Several display +# names can collapse to one canonical key; we union their HF ids. +_AA_CANON_TO_HF_IDS: dict[str, list[str]] = {} +for _disp, _ids in AA_NAME_TO_HF_IDS.items(): + _AA_CANON_TO_HF_IDS.setdefault(_canonical_name(_disp), []).extend(_ids) + + +def _decode_rsc_blob(html: str) -> str: + """Concatenate and unescape the App Router RSC chunks into one string.""" + parts: list[str] = [] + for m in _RSC_CHUNK_RE.finditer(html): + try: + parts.append(json.loads(m.group("s"))) + except (ValueError, json.JSONDecodeError): + continue + return "".join(parts) + + +def _extract_aa_pairs_from_html(html: str) -> list[tuple[str, float]]: + """Extract (name, intelligence_index) pairs from the RSC stream.""" + blob = _decode_rsc_blob(html) + if not blob: + return [] + pairs: list[tuple[str, float]] = [] + for m in _AA_RECORD_RE.finditer(blob): + try: + name = json.loads('"' + m.group("name") + '"').strip() + score = float(m.group("idx")) + except (ValueError, json.JSONDecodeError): + continue + if name and score > 0: + pairs.append((name, score)) + return pairs + + +def _extract_aa_pairs(payload: dict) -> list[tuple[str, float]]: + """Walk the Next.js payload looking for {name, intelligenceIndex}-shaped + objects regardless of where they are nested.""" + pairs: list[tuple[str, float]] = [] + for node in _walk(payload): + # Look for the most common shapes AA has used in past iterations. + name = None + score = None + for name_key in ("model_name", "modelName", "name", "displayName"): + v = node.get(name_key) + if isinstance(v, str) and v.strip(): + name = v.strip() + break + for score_key in ( + "intelligence_index", + "intelligenceIndex", + "aa_index", + "aaIndex", + "score", + ): + v = node.get(score_key) + if isinstance(v, (int, float)): + score = float(v) + break + if name and score is not None and score > 0: + pairs.append((name, score)) + return pairs + + +async def fetch_aa_index_scores(client: httpx.AsyncClient) -> dict[str, float]: + """Fetch Artificial Analysis Intelligence Index scores. + + Returns ``{hf_id: normalized_score_0_100}`` for every model AA reports + that we can map back to a HuggingFace repo via :data:`AA_NAME_TO_HF_IDS`. + + Raises on HTTP / parse failure. + """ + resp = await get_with_retries(client, AA_LEADERBOARD_URL) + resp.raise_for_status() + # Primary: Next.js App Router RSC stream (current site format). + pairs = _extract_aa_pairs_from_html(resp.text) + # Legacy fallback: classic __NEXT_DATA__ JSON blob (older site format). + if not pairs: + match = _NEXT_DATA_RE.search(resp.text) + if match: + try: + pairs = _extract_aa_pairs(json.loads(match.group("json"))) + except (ValueError, json.JSONDecodeError): + pairs = [] + if not pairs: + raise ExtractionFailed( + "AA leaderboard: no (name, score) pairs found " + "(neither RSC __next_f nor __NEXT_DATA__ matched)" + ) + # When the same display name appears multiple times (different size / + # reasoning tiers), keep the maximum value — it represents the most + # capable variant available. + best_by_name: dict[str, float] = {} + for name, score in pairs: + current = best_by_name.get(name) + if current is None or score > current: + best_by_name[name] = score + + live: dict[str, float] = {} + for name, score in best_by_name.items(): + # Exact display name first, then canonicalized (variant-stripped) match. + hf_ids = AA_NAME_TO_HF_IDS.get(name) or _AA_CANON_TO_HF_IDS.get( + _canonical_name(name) + ) + if not hf_ids: + continue + normalized = _normalize_aa_index(score) + if normalized <= 0: + continue + for hf_id in hf_ids: + if normalized > live.get(hf_id, 0.0): + live[hf_id] = normalized + if not live: + raise ExtractionFailed("AA index: live fetch returned 0 mapped scores") + + # Overlay live scores on top of the curated snapshot so a successful live + # fetch can only ADD coverage, never shrink it below the fallback. Live + # numbers win wherever both exist; the snapshot fills the long tail of + # models AA labels in a way we can't map (or no longer tracks). + scores = get_aa_curated_fallback() + for hf_id, normalized in live.items(): + if normalized > scores.get(hf_id, 0.0): + scores[hf_id] = normalized + logger.debug(f"AA index: {len(live)} live + {len(scores)} merged scores") + return scores + + +def get_aa_curated_fallback() -> dict[str, float]: + """Return the 2026-06-29 curated snapshot, normalized to the 0-100 scale. + + Used whenever the live HTML scrape cannot extract data — for example + when artificialanalysis.ai changes its Next.js payload shape. + """ + result: dict[str, float] = {} + for hf_id, raw in AA_INDEX_FALLBACK_2026_06_29.items(): + normalized = _normalize_aa_index(raw) + if normalized > 0: + result[hf_id] = normalized + return result diff --git a/src/whichllm/models/benchmark_sources/aider.py b/src/whichllm/models/benchmark_sources/aider.py new file mode 100644 index 0000000..d807c74 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/aider.py @@ -0,0 +1,135 @@ +"""Aider polyglot leaderboard source. + +The polyglot leaderboard ranks LLMs by how well they edit code across six +languages (C++, Go, Java, JavaScript, Python, Rust). Aider publishes the raw +data as a YAML file in its GitHub repo, which is easier to parse than the +rendered HTML and rarely changes shape. + +Used primarily for ``profile=coding`` ranking, but the score is also merged +into the general benchmark dict so coding-strong models get a small bump +elsewhere. +""" + +from __future__ import annotations + +import logging +import re + +import httpx + +from whichllm.models.http import get_with_retries + +logger = logging.getLogger(__name__) + +AIDER_POLYGLOT_YML_URL = ( + "https://raw.githubusercontent.com/Aider-AI/aider/main/" + "aider/website/_data/polyglot_leaderboard.yml" +) + +# Polyglot pass-rate is 0-100 (percent of exercises passing). Treat the +# floor/ceiling as 0..90 since the cap of practical models is ~88%. +_PG_MIN = 0.0 +_PG_MAX = 90.0 + +AIDER_NAME_TO_HF_IDS: dict[str, list[str]] = { + "deepseek-r1": ["deepseek-ai/DeepSeek-R1"], + "deepseek-r1-0528": ["deepseek-ai/DeepSeek-R1-0528"], + "deepseek-v3": ["deepseek-ai/DeepSeek-V3"], + "deepseek-v3-0324": ["deepseek-ai/DeepSeek-V3-0324"], + "deepseek-v3.1": ["deepseek-ai/DeepSeek-V3.1"], + "deepseek-v3.2": ["deepseek-ai/DeepSeek-V3.2"], + "deepseek-v4-pro": ["deepseek-ai/DeepSeek-V4-Pro"], + "deepseek-v4-flash": ["deepseek-ai/DeepSeek-V4-Flash"], + "qwen3-coder-30b-a3b-instruct": ["Qwen/Qwen3-Coder-30B-A3B-Instruct"], + "qwen3-coder-next": ["Qwen/Qwen3-Coder-Next"], + "qwen2.5-coder-32b-instruct": ["Qwen/Qwen2.5-Coder-32B-Instruct"], + "qwen3-32b": ["Qwen/Qwen3-32B"], + "qwen3.6-27b": ["Qwen/Qwen3.6-27B"], + "llama-3.3-70b-instruct": ["meta-llama/Llama-3.3-70B-Instruct"], + "llama-4-maverick": ["meta-llama/Llama-4-Maverick-17B-128E-Instruct"], + "gemma-3-27b-it": ["google/gemma-3-27b-it"], + "gemma-4-31b": ["google/gemma-4-31b-it"], + "mistral-large-2411": ["mistralai/Mistral-Large-Instruct-2411"], + "devstral-small": ["mistralai/Devstral-Small-2505"], + "gpt-oss-120b": ["openai/gpt-oss-120b"], + "gpt-oss-20b": ["openai/gpt-oss-20b"], + "glm-4.5": ["zai-org/GLM-4.5"], + "glm-4.6": ["zai-org/GLM-4.6"], + "glm-5": ["zai-org/GLM-5"], + "glm-5.1": ["zai-org/GLM-5.1"], + "kimi-k2-instruct": ["moonshotai/Kimi-K2-Instruct"], + "phi-4": ["microsoft/phi-4"], + "qwq-32b": ["Qwen/QwQ-32B"], +} + + +_PASS_RATE_RE = re.compile(r"pass_rate[_-]?2[:\s]+(\d+(?:\.\d+)?)", re.IGNORECASE) +_MODEL_RE = re.compile(r"^- model[:\s]+(.+)$", re.MULTILINE) + + +def _normalize(pass_rate: float) -> float: + if not isinstance(pass_rate, (int, float)): + return 0.0 + span = _PG_MAX - _PG_MIN + normalized = (pass_rate - _PG_MIN) / span * 100.0 + return max(0.0, min(100.0, round(normalized, 1))) + + +def _parse_yaml_lite(text: str) -> list[tuple[str, float]]: + """Tiny YAML extractor for the polyglot leaderboard format. + + We avoid pulling in PyYAML; the file shape is stable enough that two + regexes scanning each record block suffice. Each record looks like: + - dirname: 2024-12-22-blah + model: deepseek/deepseek-chat + edit_format: diff + pass_rate_2: 80.7 + ... + """ + out: list[tuple[str, float]] = [] + # Split into records starting with "- " + records = re.split(r"\n(?=-\s+\w)", text) + for rec in records: + m_model = re.search(r"^\s*model[:\s]+(.+?)$", rec, re.MULTILINE | re.IGNORECASE) + m_rate = re.search(r"pass_rate_2[:\s]+(\d+(?:\.\d+)?)", rec, re.IGNORECASE) + if not m_model or not m_rate: + continue + name = m_model.group(1).strip().strip("\"'") + # Strip any provider prefix like "deepseek/" or "openrouter/" + name = name.split("/", 1)[-1].strip().lower() + try: + rate = float(m_rate.group(1)) + except ValueError: + continue + if rate <= 0: + continue + out.append((name, rate)) + return out + + +async def fetch_aider_polyglot_scores(client: httpx.AsyncClient) -> dict[str, float]: + """Fetch Aider polyglot pass-rates. Raises on HTTP / parse failure.""" + scores: dict[str, float] = {} + resp = await get_with_retries(client, AIDER_POLYGLOT_YML_URL) + resp.raise_for_status() + pairs = _parse_yaml_lite(resp.text) + if not pairs: + logger.debug("Aider polyglot: 0 records parsed") + return {} + best_by_name: dict[str, float] = {} + for name, rate in pairs: + cur = best_by_name.get(name) + if cur is None or rate > cur: + best_by_name[name] = rate + for name, rate in best_by_name.items(): + ids = AIDER_NAME_TO_HF_IDS.get(name) + if not ids: + continue + normalized = _normalize(rate) + if normalized <= 0: + continue + for hf_id in ids: + if scores.get(hf_id, 0.0) < normalized: + scores[hf_id] = normalized + logger.debug(f"Aider polyglot: {len(scores)} mapped scores") + return scores diff --git a/src/whichllm/models/benchmark_sources/chatbot_arena.py b/src/whichllm/models/benchmark_sources/chatbot_arena.py new file mode 100644 index 0000000..c9c8cc7 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/chatbot_arena.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import re + +import httpx + +from whichllm.models.http import get_with_retries + +# --- Data source URLs --- +ARENA_ROWS_URL = "https://datasets-server.huggingface.co/rows" +ARENA_DATASET = "mathewhe/chatbot-arena-elo" + +# --- Arena ELO normalization --- +# Open-source ELO range: ~1030 (worst) to ~1424 (best). Arena is frozen +# 2025-07-17 (no new models added) so the leaderboard cannot reflect any +# 2025-Q3+ release; we cap the normalized output at 82 so newer benchmark +# sources (AA Index / LiveBench, which can reach 95+) decisively win on +# conflict. +_ARENA_ELO_MIN = 1030 +_ARENA_ELO_MAX = 1430 +_ARENA_MAX_NORMALIZED = 82.0 + +# --- Arena display name -> HuggingFace org mapping --- +_ARENA_ORG_TO_HF: dict[str, list[str]] = { + "Alibaba": ["Qwen"], + "Meta": ["meta-llama"], + "DeepSeek": ["deepseek-ai"], + "DeepSeek AI": ["deepseek-ai"], + "Google": ["google"], + "Mistral": ["mistralai"], + "Microsoft": ["microsoft"], + "Nvidia": ["nvidia"], + "01 AI": ["01-ai"], + "Allen AI": ["allenai"], + "Ai2": ["allenai"], + "AllenAI/UW": ["allenai"], + "Cohere": ["CohereForAI"], + "HuggingFace": ["HuggingFaceH4", "huggingface"], + "AI21 Labs": ["ai21labs"], + "NousResearch": ["NousResearch"], + "NexusFlow": ["Nexusflow"], + "Princeton": ["princeton-nlp"], + "IBM": ["ibm-granite"], + "InternLM": ["internlm"], + "Together AI": ["togethercomputer"], + "TII": ["tiiuae"], + "MiniMax": ["MiniMaxAI"], + "MosaicML": ["mosaicml"], + "Databricks": ["databricks"], + "Moonshot": ["moonshotai"], + "UC Berkeley": ["berkeley-nest"], + "Cognitive Computations": ["cognitivecomputations"], + "Upstage AI": ["upstage"], + "UW": ["timdettmers"], + "Snowflake": ["Snowflake"], + "LMSYS": ["lmsys"], + "OpenChat": ["openchat"], +} + + +def _normalize_arena_elo(elo: float) -> float: + """Normalize Arena ELO to a frozen-source-aware 0-_ARENA_MAX_NORMALIZED scale.""" + score = ( + (elo - _ARENA_ELO_MIN) + / (_ARENA_ELO_MAX - _ARENA_ELO_MIN) + * _ARENA_MAX_NORMALIZED + ) + return max(0.0, min(_ARENA_MAX_NORMALIZED, round(score, 1))) + + +def _arena_name_to_hf_ids(model_name: str, org: str) -> list[str]: + """Convert Arena display name to potential HuggingFace model IDs.""" + hf_orgs = _ARENA_ORG_TO_HF.get(org, []) + candidates = [] + + # Clean the model name: remove date suffixes like "(03-2025)" + clean_name = re.sub(r"\s*\([\d-]+\)\s*$", "", model_name).strip() + # Remove -bf16, -fp8 suffixes for base matching + base_name = re.sub(r"-(bf16|fp8|fp16)$", "", clean_name, flags=re.IGNORECASE) + + for hf_org in hf_orgs: + candidates.append(f"{hf_org}/{clean_name}") + if base_name != clean_name: + candidates.append(f"{hf_org}/{base_name}") + # Try with -Instruct suffix stripped for base model matching + no_instruct = re.sub(r"-Instruct$", "", clean_name) + if no_instruct != clean_name: + candidates.append(f"{hf_org}/{no_instruct}") + + return candidates + + +async def fetch_arena_scores(client: httpx.AsyncClient) -> dict[str, float]: + """Fetch Chatbot Arena ELO scores via rows API.""" + scores: dict[str, float] = {} + offset = 0 + + while True: + resp = await get_with_retries( + client, + ARENA_ROWS_URL, + params={ + "dataset": ARENA_DATASET, + "config": "default", + "split": "train", + "offset": str(offset), + "length": "100", + }, + ) + resp.raise_for_status() + data = resp.json() + rows = data.get("rows", []) + if not rows: + break + + for r in rows: + row = r.get("row", {}) + model_name = str(row.get("Model", "")) + elo = row.get("Arena Score", 0) + org = str(row.get("Organization", "")) + lic = str(row.get("License", "")) + + if not model_name or not elo or elo <= 0: + continue + # Skip proprietary models (can't run locally) + if "Proprietary" in lic or "Propretary" in lic: + continue + + normalized = _normalize_arena_elo(elo) + # Map to all potential HF IDs + hf_ids = _arena_name_to_hf_ids(model_name, org) + for hf_id in hf_ids: + scores[hf_id] = normalized + + offset += len(rows) + total = data.get("num_rows_total", 0) + if total and offset >= total: + break + + return scores diff --git a/src/whichllm/models/benchmark_sources/constants.py b/src/whichllm/models/benchmark_sources/constants.py new file mode 100644 index 0000000..1e3ba11 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/constants.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import re + +_NEXT_DATA_RE = re.compile( + r'', re.DOTALL +) diff --git a/src/whichllm/models/benchmark_sources/livebench.py b/src/whichllm/models/benchmark_sources/livebench.py new file mode 100644 index 0000000..0a96239 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/livebench.py @@ -0,0 +1,102 @@ +"""LiveBench (livebench.ai) source — inlined snapshot. + +LiveBench publishes its leaderboard as a dated CSV (e.g. +``https://livebench.ai/table_2026_01_08.csv``). +To refresh: download the latest ``table_YYYY_MM_DD.csv`` from livebench.ai +and run ``scripts/import_livebench_csv.py`` to regenerate the dict below. +""" + +from __future__ import annotations + +# Inlined LiveBench global-average snapshot. Values are raw 0-100; the +# normalizer below rescales them onto the project's shared 0-100 axis. +# Entries from the 2026-01-08 CSV were generated by ``scripts/import_livebench_csv.py``. +LIVEBENCH_RAW_DATA: dict[str, float] = { + # --- 2026-01-08 CSV (auto-generated; see scripts/import_livebench_csv.py) + "MiniMaxAI/MiniMax-M2.5": 60.3, + "MiniMaxAI/MiniMax-M2.7": 65.0, + "Qwen/Qwen3-235B-A22B-Instruct-2507": 48.0, + "Qwen/Qwen3-235B-A22B-Thinking-2507": 52.9, + "Qwen/Qwen3-30B-A3B-Thinking-2507": 38.8, + "Qwen/Qwen3-32B": 42.7, + "Qwen/Qwen3-Next-80B-A3B-Instruct": 47.4, + "Qwen/Qwen3-Next-80B-A3B-Thinking": 51.0, + "Qwen/Qwen3.6-27B": 65.6, + "XiaomiMiMo/MiMo-V2-Pro": 58.4, + "deepseek-ai/DeepSeek-V3.2": 63.1, + "deepseek-ai/DeepSeek-V3.2-Exp": 58.9, + "deepseek-ai/DeepSeek-V4-Flash": 67.7, + "deepseek-ai/DeepSeek-V4-Pro": 74.4, + "google/gemma-4-31b-it": 62.4, + "mistralai/Devstral-2512": 38.8, + "moonshotai/Kimi-K2-Instruct": 45.9, + "moonshotai/Kimi-K2-Thinking": 62.3, + "moonshotai/Kimi-K2.5": 69.2, + "moonshotai/Kimi-K2.6-Thinking": 72.4, + "nvidia/Nemotron-3-Super-120B-A12B": 32.0, + "openai/gpt-oss-120b": 46.4, + "zai-org/GLM-4.6": 54.7, + "zai-org/GLM-4.6V": 38.9, + "zai-org/GLM-4.7": 57.3, + "zai-org/GLM-5": 68.7, + "zai-org/GLM-5.1": 70.6, + # --- 2026-04 carryover (anchors for older / smaller-class models) + "deepseek-ai/DeepSeek-R1-0528": 71.0, + "deepseek-ai/DeepSeek-R1": 65.0, + "deepseek-ai/DeepSeek-V3-0324": 57.0, + "Qwen/Qwen3-235B-A22B": 65.0, + "Qwen/Qwen3-Coder-30B-A3B-Instruct": 58.0, + "Qwen/QwQ-32B": 57.0, + "Qwen/Qwen3-4B-Thinking-2507": 50.0, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": 56.0, + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": 50.0, + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B": 42.0, + "meta-llama/Llama-3.3-70B-Instruct": 48.0, + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 54.0, + "meta-llama/Llama-4-Scout-17B-16E-Instruct": 49.0, + "google/gemma-3-27b-it": 50.0, + "google/gemma-4-26b-a4b-it": 54.0, + "microsoft/phi-4": 53.0, + "mistralai/Mistral-Large-Instruct-2411": 58.0, + "mistralai/Devstral-Small-2505": 50.0, + "openai/gpt-oss-20b": 52.0, + "zai-org/GLM-4.5": 58.0, + "zai-org/GLM-4.5-Air": 52.0, + # 8B-class entries to anchor the smaller-model scoring + "Qwen/Qwen3-8B": 50.0, + "Qwen/Qwen3-14B": 56.0, + "Qwen/Qwen3-4B-Instruct-2507": 45.0, + "Qwen/Qwen3-4B": 42.0, + "Qwen/Qwen3-30B-A3B": 58.0, + "Qwen/Qwen2.5-7B-Instruct": 38.0, + "Qwen/Qwen2.5-14B-Instruct": 42.0, + "Qwen/Qwen2.5-32B-Instruct": 50.0, + "meta-llama/Llama-3.1-8B-Instruct": 36.0, + "google/gemma-2-9b-it": 38.0, + "google/gemma-3-12b-it": 44.0, + "microsoft/Phi-4-mini-instruct": 40.0, + "mistralai/Mistral-Small-3.2-24B-Instruct-2506": 50.0, + "mistralai/Mistral-Small-3.1-24B-Instruct-2503": 48.0, +} + +# LiveBench global-average tops out around 72 for current frontier models +# (DeepSeek V4 Pro 72, Kimi K2.6 71, Qwen3.6-27B 66) with 8B-class around 35. +# Anchored by a two-point fit: top frontier (72) → 95, mid/8B (35) → 30 — so +# 8B models get a meaningful but not dominant share and frontier MoE clears +# the OLLB cap. +_LB_MIN = 18.0 +_LB_MAX = 75.0 + + +def _normalize_livebench(score: float) -> float: + normalized = (score - _LB_MIN) / (_LB_MAX - _LB_MIN) * 100.0 + return max(0.0, min(100.0, round(normalized, 1))) + + +def get_livebench_data() -> dict[str, float]: + """Return the inlined LiveBench snapshot, normalized to the 0-100 scale.""" + return { + hf_id: _normalize_livebench(raw) + for hf_id, raw in LIVEBENCH_RAW_DATA.items() + if _normalize_livebench(raw) > 0 + } diff --git a/src/whichllm/models/benchmark_sources/open_llm_leaderboard.py b/src/whichllm/models/benchmark_sources/open_llm_leaderboard.py new file mode 100644 index 0000000..1c616ba --- /dev/null +++ b/src/whichllm/models/benchmark_sources/open_llm_leaderboard.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import io + +import httpx + +from whichllm.models.http import get_with_retries + +LEADERBOARD_PARQUET_URL = ( + "https://huggingface.co/api/datasets/open-llm-leaderboard/contents" + "/parquet/default/train/0.parquet" +) +LEADERBOARD_ROWS_URL = "https://datasets-server.huggingface.co/rows" +LEADERBOARD_DATASET = "open-llm-leaderboard/contents" + +# --- Leaderboard normalization --- +# OLLB v2 averages range ~5 to ~52. The leaderboard is archived 2025-06 with +# the top slot held by Qwen2.5-32B (47.6 raw = 91.5 if uncapped); capping at +# 78 prevents an older generation with a strong-but-frozen OLLB score from +# dominating rankings that now have AA Index / LiveBench coverage too. +_LB_AVG_MAX = 52 +_OLLB_MAX_NORMALIZED = 78.0 + + +async def _fetch_leaderboard_parquet(client: httpx.AsyncClient) -> dict[str, float]: + """Download Open LLM Leaderboard parquet (requires pyarrow).""" + import pyarrow.parquet as pq + + resp = await get_with_retries( + client, LEADERBOARD_PARQUET_URL, follow_redirects=True + ) + resp.raise_for_status() + table = pq.read_table( + io.BytesIO(resp.content), + columns=["fullname", "Average ⬆️"], + ) + d = table.to_pydict() + scores: dict[str, float] = {} + for i in range(len(d["fullname"])): + name = d["fullname"][i] + avg = d["Average ⬆️"][i] + if name and avg and avg > 0: + scores[name] = _normalize_leaderboard_avg(avg) + return scores + + +async def _fetch_leaderboard_api(client: httpx.AsyncClient) -> dict[str, float]: + """Fetch Open LLM Leaderboard via rows API (no pyarrow needed).""" + scores: dict[str, float] = {} + offset = 0 + + while True: + resp = await get_with_retries( + client, + LEADERBOARD_ROWS_URL, + params={ + "dataset": LEADERBOARD_DATASET, + "config": "default", + "split": "train", + "offset": str(offset), + "length": "100", + }, + ) + resp.raise_for_status() + data = resp.json() + rows = data.get("rows", []) + if not rows: + break + + for r in rows: + row = r.get("row", {}) + name = row.get("fullname") + avg = row.get("Average ⬆️") + if name and avg and avg > 0: + scores[name] = _normalize_leaderboard_avg(avg) + + offset += len(rows) + total = data.get("num_rows_total", 0) + if total and offset >= total: + break + + return scores + + +def _normalize_leaderboard_avg(avg: float) -> float: + """Normalize Open LLM Leaderboard average to 0-_OLLB_MAX_NORMALIZED scale.""" + score = avg / _LB_AVG_MAX * _OLLB_MAX_NORMALIZED + return max(0.0, min(_OLLB_MAX_NORMALIZED, round(score, 1))) + + +async def fetch_leaderboard_with_fallback( + client: httpx.AsyncClient, +) -> dict[str, float]: + """Prefer the parquet path (one request, full table) and fall back to the + paginated rows API when pyarrow is unavailable.""" + try: + return await _fetch_leaderboard_parquet(client) + except ImportError: + return await _fetch_leaderboard_api(client) diff --git a/src/whichllm/models/benchmark_sources/types.py b/src/whichllm/models/benchmark_sources/types.py new file mode 100644 index 0000000..06389f0 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/types.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class ExtractionFailed(Exception): + pass diff --git a/src/whichllm/models/benchmark_sources/utils.py b/src/whichllm/models/benchmark_sources/utils.py new file mode 100644 index 0000000..cd77a98 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/utils.py @@ -0,0 +1,14 @@ +from __future__ import annotations + + +def _walk(obj, depth: int = 0): + """Yield every dict encountered while recursively walking a JSON tree.""" + if depth > 12: + return + if isinstance(obj, dict): + yield obj + for v in obj.values(): + yield from _walk(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + yield from _walk(item, depth + 1) diff --git a/src/whichllm/models/benchmark_sources/vision.py b/src/whichllm/models/benchmark_sources/vision.py new file mode 100644 index 0000000..e422232 --- /dev/null +++ b/src/whichllm/models/benchmark_sources/vision.py @@ -0,0 +1,83 @@ +"""Vision-language model benchmark source. + +Text leaderboards (LiveBench, AA Index, Aider) do not score VLMs, so a +``--profile vision`` ranking had only one model with a ``direct`` hit +(Qwen2-VL-7B, a two-generations-old model) and every current VLM fell +back to size/popularity heuristics — letting the 8B legacy model +outrank Qwen3-VL-32B even on an 80 GB H100. + +There is no single stable machine-readable VLM leaderboard (the MMMU / +OpenCompass spaces change their JSON shape frequently), so this source +is a curated snapshot rather than a live scrape. Values are a blended +0-100 capability index drawn from MMMU-Pro, MMBench, and general +multimodal evaluations as of 2026-05; they are already normalized and +merged into the combined benchmark dict like any other current source. +Profile filtering keeps these scores from affecting text rankings — +only models tagged ``vision`` consume them. +""" + +from __future__ import annotations + +import logging + +import httpx + +logger = logging.getLogger(__name__) + +# Curated multimodal capability index (0-100), 2026-05 snapshot. +# Anchored so the current Qwen3-VL / InternVL3 frontier sits in the +# mid-50s-to-60s and two-generations-old 7B-class VLMs sit in the low +# 30s, which restores the correct generational ordering. +VISION_FALLBACK_2026_05: dict[str, float] = { + # Qwen3-VL (current frontier) + "Qwen/Qwen3-VL-235B-A22B-Instruct": 62.0, + "Qwen/Qwen3-VL-32B-Instruct": 57.0, + "Qwen/Qwen3-VL-30B-A3B-Instruct": 53.0, + "Qwen/Qwen3-VL-8B-Instruct": 45.0, + "Qwen/Qwen3-VL-8B-Thinking": 46.0, + "Qwen/Qwen3-VL-4B-Instruct": 37.0, + "Qwen/Qwen3-VL-4B-Thinking": 38.0, + # Qwen2.5-VL (previous generation, still strong) + "Qwen/Qwen2.5-VL-72B-Instruct": 55.0, + "Qwen/Qwen2.5-VL-32B-Instruct": 49.0, + "Qwen/Qwen2.5-VL-7B-Instruct": 41.0, + "Qwen/Qwen2.5-VL-3B-Instruct": 33.0, + # Qwen2-VL (two generations old — must rank below Qwen3-VL) + "Qwen/Qwen2-VL-72B-Instruct": 45.0, + "Qwen/Qwen2-VL-7B-Instruct": 33.0, + "Qwen/Qwen2-VL-2B-Instruct": 24.0, + # Meta Llama Vision + "meta-llama/Llama-3.2-90B-Vision-Instruct": 41.0, + "meta-llama/Llama-3.2-11B-Vision-Instruct": 29.0, + # Microsoft Phi vision + "microsoft/Phi-4-reasoning-vision-15B": 46.0, + "microsoft/Phi-3.5-vision-instruct": 27.0, + # Google Gemma 3 (natively multimodal) + "google/gemma-3-27b-it": 42.0, + "google/gemma-3-12b-it": 35.0, + "google/gemma-3-4b-it": 27.0, + # Mistral Pixtral + "mistralai/Pixtral-12B-2409": 35.0, + "mistral-community/pixtral-12b": 35.0, + # OpenGVLab InternVL3 (frontier open VLM) + "OpenGVLab/InternVL3-78B": 56.0, + "OpenGVLab/InternVL3-38B": 52.0, + "OpenGVLab/InternVL3-14B": 45.0, + "OpenGVLab/InternVL3-8B": 40.0, + "OpenGVLab/InternVL2_5-78B": 50.0, + # DeepSeek-VL2 + "deepseek-ai/deepseek-vl2": 38.0, + # zhipu / GLM vision + "zai-org/GLM-4.5V": 50.0, +} + + +async def fetch_vision_scores(client: httpx.AsyncClient) -> dict[str, float]: + """Return curated VLM capability scores. + + No stable live source exists, so this returns the frozen snapshot. + The ``client`` parameter mirrors the other source functions so the + caller can treat all sources uniformly and a live scrape can be + slotted in later without changing call sites. + """ + return dict(VISION_FALLBACK_2026_05) diff --git a/src/whichllm/models/benchmark_types.py b/src/whichllm/models/benchmark_types.py new file mode 100644 index 0000000..cbaa4b5 --- /dev/null +++ b/src/whichllm/models/benchmark_types.py @@ -0,0 +1,23 @@ +"""Shared benchmark data types.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BenchmarkEvidence: + """Benchmark evidence with confidence. + + source values, ordered from most trusted to least: + - "direct" : independent leaderboard / Arena ELO hit on exact id + - "variant" : suffix-stripped derivative of a direct leaderboard hit + - "base_model" : cardData.base_model pointer to a direct hit + - "line_interp" : size-aware interpolation within the same model line + - "self_reported" : evalResults reported by the uploader themselves + - "none" : no usable signal + """ + + score: float | None + confidence: float + source: str diff --git a/src/whichllm/models/cache.py b/src/whichllm/models/cache.py new file mode 100644 index 0000000..8e636ab --- /dev/null +++ b/src/whichllm/models/cache.py @@ -0,0 +1,47 @@ +"""Local JSON cache with TTL for model data.""" + +from __future__ import annotations + +import json +import logging +import time + +from whichllm.utils import _cache_dir + +logger = logging.getLogger(__name__) + +CACHE_DIR = _cache_dir() +CACHE_FILE = CACHE_DIR / "models.json" +DEFAULT_TTL_SECONDS = 6 * 3600 # 6 hours + + +def _ensure_cache_dir() -> None: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + + +def load_cache() -> list[dict] | None: + """Load cached model data if valid. Returns None if expired or missing.""" + if not CACHE_FILE.exists(): + return None + + try: + data = json.loads(CACHE_FILE.read_text(encoding="utf-8")) + cached_at = data.get("cached_at", 0) + if time.time() - cached_at > DEFAULT_TTL_SECONDS: + logger.debug("Cache expired") + return None + return data.get("models", []) + except (json.JSONDecodeError, KeyError) as e: + logger.debug(f"Cache corrupted: {e}") + return None + + +def save_cache(models: list[dict]) -> None: + """Save model data to cache.""" + _ensure_cache_dir() + data = { + "cached_at": time.time(), + "models": models, + } + CACHE_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + logger.debug(f"Saved {len(models)} models to cache") diff --git a/src/whichllm/models/fetcher.py b/src/whichllm/models/fetcher.py new file mode 100644 index 0000000..02b285d --- /dev/null +++ b/src/whichllm/models/fetcher.py @@ -0,0 +1,111 @@ +"""Compatibility shim for HuggingFace model fetching helpers. + +The implementation is split by responsibility across ``whichllm.models``: + +- ``hf`` for HuggingFace API orchestration +- ``parser`` for API payload to ModelInfo conversion +- ``parameters`` for parameter-count and MoE normalization +- ``gguf`` for GGUF filename/variant parsing +- ``sliding_window`` for SWA metadata resolution +- ``serialization`` for cache serialization + +Existing imports from ``whichllm.models.fetcher`` are re-exported here. +""" + +from __future__ import annotations + +from whichllm.models import hf as _hf_module +from whichllm.models.gguf import ( + _estimate_gguf_size, + _extract_gguf_variants, + _extract_quant_type, +) +from whichllm.models.hf import ( + _DEFAULT_HF_ENDPOINT, + _FRONTIER_MODEL_IDS, + _hf_api_url, +) +from whichllm.models.http import get_with_retries +from whichllm.models.parameters import ( + _AUTHORITATIVE_PARAM_COUNTS, + _KNOWN_MOE_ACTIVE_PARAMS, + _KNOWN_PARAM_COUNTS, + _extract_active_size_hint_from_id, + _extract_size_hint_from_id, + _is_quantized_repo_name, + _lookup_curated_count, + _normalize_param_count, + _resolve_moe_active_params, +) +from whichllm.models.parser import ( + _extract_architecture, + _extract_hf_eval_score, + _extract_param_count, + _extract_published_at, + _is_general_eval_entry, + _normalize_eval_value, + _parse_model, +) +from whichllm.models.serialization import dicts_to_models, models_to_dicts +from whichllm.models.sliding_window import ( + _SWA_ARCH_ALIASES, + _SWA_ARCH_DEFAULTS, + _resolve_sliding_window, + _swa_arch_key, + _swa_key_from_arch, +) + + +async def fetch_models(limit: int = 300, include_vision: bool = True): + """Fetch popular models from HuggingFace Hub. + + The wrapper keeps legacy monkey-patching of ``fetcher.get_with_retries`` + working while the implementation lives in ``whichllm.models.hf``. + """ + original = _hf_module.get_with_retries + _hf_module.get_with_retries = get_with_retries + try: + return await _hf_module.fetch_models(limit=limit, include_vision=include_vision) + finally: + _hf_module.get_with_retries = original + + +async def fetch_model_published_at(model_ids: list[str]): + """Fetch published timestamps for specific model IDs.""" + return await _hf_module.fetch_model_published_at(model_ids) + + +__all__ = [ + "_AUTHORITATIVE_PARAM_COUNTS", + "_DEFAULT_HF_ENDPOINT", + "_FRONTIER_MODEL_IDS", + "_KNOWN_MOE_ACTIVE_PARAMS", + "_KNOWN_PARAM_COUNTS", + "_SWA_ARCH_ALIASES", + "_SWA_ARCH_DEFAULTS", + "_estimate_gguf_size", + "_extract_active_size_hint_from_id", + "_extract_architecture", + "_extract_gguf_variants", + "_extract_hf_eval_score", + "_extract_param_count", + "_extract_published_at", + "_extract_quant_type", + "_extract_size_hint_from_id", + "_hf_api_url", + "_is_general_eval_entry", + "_is_quantized_repo_name", + "_lookup_curated_count", + "_normalize_eval_value", + "_normalize_param_count", + "_parse_model", + "_resolve_moe_active_params", + "_resolve_sliding_window", + "_swa_arch_key", + "_swa_key_from_arch", + "dicts_to_models", + "fetch_model_published_at", + "fetch_models", + "get_with_retries", + "models_to_dicts", +] diff --git a/src/whichllm/models/gguf.py b/src/whichllm/models/gguf.py new file mode 100644 index 0000000..ae8ef89 --- /dev/null +++ b/src/whichllm/models/gguf.py @@ -0,0 +1,84 @@ +"""GGUF filename parsing and variant extraction helpers.""" + +from __future__ import annotations + +import re + +from whichllm.constants import QUANT_BYTES_PER_WEIGHT +from whichllm.models.types import GGUFVariant + +_GGUF_SPLIT_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE) + +# Filename quant tokens that name the same format under a different spelling +# than the canonical key used by QUANT_BYTES_PER_WEIGHT / QUANT_QUALITY_PENALTY. +_QUANT_ALIASES = { + "FP16": "F16", + "FP32": "F32", +} + + +def _extract_quant_type(filename: str) -> str: + """Extract quantization type from a GGUF filename. + + The returned key is canonicalized to match QUANT_BYTES_PER_WEIGHT, so + callers can look it up directly. Returns "unknown" when nothing matches. + """ + patterns = [ + r"[.-](Q\d+_K_[SMLA])", + r"[.-](Q\d+_\d+)", + r"[.-](Q\d+_K)", + r"[.-](TQ\d+_\d+)", + r"[.-](IQ\d+_\w+)", + r"[.-](MXFP4|NVFP4)", + r"[.-](F16|FP16|BF16|F32|FP32)", + ] + upper = filename.upper() + for pattern in patterns: + m = re.search(pattern, upper) + if m: + quant = m.group(1) + return _QUANT_ALIASES.get(quant, quant) + return "unknown" + + +def _estimate_gguf_size(param_count: int, quant_type: str) -> int: + """Estimate GGUF file size from parameter count and quantization type.""" + bpw = QUANT_BYTES_PER_WEIGHT.get(quant_type.upper(), 0.5625) # default Q4_K_M + return int(param_count * bpw) + + +def _extract_gguf_variants(data: dict, param_count: int) -> list[GGUFVariant]: + """Extract GGUF variants from HF sibling metadata.""" + quant_sizes: dict[str, int] = {} + quant_first_filename: dict[str, str] = {} + siblings = data.get("siblings", []) or [] + for sib in siblings: + fname = sib.get("rfilename", "") + if not fname.endswith(".gguf") or fname.startswith("."): + continue + quant = _extract_quant_type(fname) + if quant == "unknown": + continue + size = sib.get("size", 0) + if not isinstance(size, int) or size < 0: + size = 0 + + # Split GGUF files are summed into one candidate per quant. + quant_sizes[quant] = quant_sizes.get(quant, 0) + size + if quant not in quant_first_filename or _GGUF_SPLIT_RE.search( + quant_first_filename[quant] + ): + quant_first_filename[quant] = fname + + gguf_variants = [] + for quant, total_size in quant_sizes.items(): + if total_size <= 0: + total_size = _estimate_gguf_size(param_count, quant) + gguf_variants.append( + GGUFVariant( + filename=quant_first_filename[quant], + quant_type=quant, + file_size_bytes=total_size, + ) + ) + return gguf_variants diff --git a/src/whichllm/models/grouper.py b/src/whichllm/models/grouper.py new file mode 100644 index 0000000..2d7d843 --- /dev/null +++ b/src/whichllm/models/grouper.py @@ -0,0 +1,149 @@ +"""Model family grouping logic.""" + +from __future__ import annotations + +import re + +from whichllm.models.types import ModelFamily, ModelInfo + + +def _normalize_name(model_id: str) -> str: + """Normalize model ID for grouping by removing org prefix and GGUF/quant/chat suffixes.""" + name = model_id.lower() + # Strip org prefix (e.g. "bartowski/Meta-Llama-3.1" -> "meta-llama-3.1") + if "/" in name: + name = name.split("/", 1)[1] + # Strip common org prefixes in model names (e.g. "qwen_qwen3-8b" -> "qwen3-8b") + name = re.sub(r"^(qwen_|meta-llama_|google_)", "", name) + # Remove common suffixes (applied repeatedly to handle stacked suffixes) + suffixes = [ + r"-gguf$", + r"-gptq$", + r"-awq$", + r"-instruct$", + r"-chat$", + r"-it$", + r"-hf$", + r"-fp8$", + r"-fp16$", + r"-bf16$", + r"-mxfp4$", + r"-nvfp4$", + r"-\d+bit$", + r"-\d{4}$", # date suffixes like -2507, -2503 + ] + for _ in range(3): # multiple passes to strip stacked suffixes + prev = name + for suffix in suffixes: + name = re.sub(suffix, "", name) + if name == prev: + break + + # Strip version-before-size: mistral-small-3.2-24b -> mistral-small-24b + # This catches patterns like MODEL-MAJOR.MINOR-SIZEb where the version + # is a separate segment (preceded by '-') before the size suffix. + # Does NOT match qwen3.5-27b because '3.5' is glued to 'qwen' without '-'. + name = re.sub(r"-\d+\.\d+(-\d+(?:\.\d+)?b(?:-a\d+b)?)$", r"\1", name) + + # Split series name from size suffix, strip minor version from series only. + # Merges qwen3.5-27b + qwen3-30b-a3b naming variants (different sizes stay separate). + m = re.match(r"^(.+?)-(\d+(?:\.\d+)?b(?:-a\d+b)?)$", name) + if m: + series, size = m.group(1), m.group(2) + series = re.sub(r"(\d+)\.\d+$", r"\1", series) + name = f"{series}-{size}" + else: + # No size suffix (e.g. deepseek-v3.2) — strip minor version directly + name = re.sub(r"(\d+)\.\d+$", r"\1", name) + + return name + + +def group_models(models: list[ModelInfo]) -> list[ModelFamily]: + """Group models into families based on base_model and name similarity.""" + # Pass 1: Group by base_model + base_model_groups: dict[str, list[ModelInfo]] = {} + ungrouped: list[ModelInfo] = [] + + for model in models: + if model.base_model: + key = model.base_model.lower() + base_model_groups.setdefault(key, []).append(model) + else: + ungrouped.append(model) + + # Pass 2: Group ungrouped by normalized name + name_groups: dict[str, list[ModelInfo]] = {} + for model in ungrouped: + key = _normalize_name(model.id) + name_groups.setdefault(key, []).append(model) + + # Merge base_model groups that share the same normalized name + merged_base: dict[str, list[ModelInfo]] = {} + for key, group in base_model_groups.items(): + norm_key = _normalize_name(key) + merged_base.setdefault(norm_key, []).extend(group) + + # Also merge with ungrouped via name matching + for norm_key, group in list(merged_base.items()): + if norm_key in name_groups: + group.extend(name_groups.pop(norm_key)) + + # Replace base_model_groups with merged version + base_model_groups = merged_base + + # Build families + families: list[ModelFamily] = [] + + for group_key, group in list(base_model_groups.items()) + list(name_groups.items()): + if not group: + continue + + # Pick the base model. Priority order: + # 1. Models that are referenced by another group member's base_model + # field — these are upstream of the others, so they are the + # true base even when a downstream fine-tune (e.g. + # prefeitura-rio/Rio-3.0-Open-Mini) has more downloads than the + # official base (Qwen/Qwen3-4B-Thinking-2507). + # 2. Models without GGUF/quant suffixes and no base_model of their + # own (the original checkpoint). + # 3. Anything left in the group. + # Within the chosen tier, pick highest downloads as a tiebreaker. + referenced_as_base: set[str] = {m.base_model for m in group if m.base_model} + upstream_candidates = [m for m in group if m.id in referenced_as_base] + if upstream_candidates: + base_candidates = upstream_candidates + else: + base_candidates = [ + m for m in group if not m.gguf_variants or m.base_model is None + ] + if not base_candidates: + base_candidates = group + + base = max(base_candidates, key=lambda m: m.downloads) + variants = [m for m in group if m.id != base.id] + + # Set family_id on all members + family_id = _normalize_name(base.id) + base.family_id = family_id + for v in variants: + v.family_id = family_id + + # Collect best benchmark scores across family + best_bench: dict[str, float] = {} + for m in group: + for k, v in m.benchmark_scores.items(): + if k not in best_bench or v > best_bench[k]: + best_bench[k] = v + + families.append( + ModelFamily( + family_id=family_id, + display_name=base.name, + base_model=base, + variants=variants, + best_benchmark=best_bench, + ) + ) + + return families diff --git a/src/whichllm/models/hf.py b/src/whichllm/models/hf.py new file mode 100644 index 0000000..3e4ae84 --- /dev/null +++ b/src/whichllm/models/hf.py @@ -0,0 +1,293 @@ +"""HuggingFace Hub client orchestration.""" + +from __future__ import annotations + +import asyncio +import logging +import os + +import httpx + +from whichllm.models.http import DEFAULT_ACCEPT_ENCODING, get_with_retries +from whichllm.models.parser import _extract_published_at, _parse_model +from whichllm.models.types import ModelInfo + +logger = logging.getLogger(__name__) + +_DEFAULT_HF_ENDPOINT = "https://huggingface.co" + +_MODEL_EXPANDS = [ + "config", + "safetensors", + "gguf", + "cardData", + "siblings", + "evalResults", +] + +_MODEL_DETAIL_EXPANDS = [ + "config", + "safetensors", + "gguf", + "cardData", + "siblings", + "evalResults", + "downloads", + "likes", + "createdAt", + "lastModified", +] + +_FRONTIER_MODEL_IDS = ( + # Newest releases that lead 2026-Q2 benchmarks + "moonshotai/Kimi-K2-Thinking", + "moonshotai/Kimi-K2-Instruct", + "moonshotai/Kimi-K2-Instruct-0905", + "XiaomiMiMo/MiMo-V2.5-Pro", + "XiaomiMiMo/MiMo-V2.5", + "XiaomiMiMo/MiMo-V2-Flash", + "deepseek-ai/DeepSeek-V4-Pro", + "deepseek-ai/DeepSeek-V4-Flash", + "deepseek-ai/DeepSeek-V3.2", + "deepseek-ai/DeepSeek-V3.2-Exp", + "deepseek-ai/DeepSeek-V3.1", + "deepseek-ai/DeepSeek-R1-0528", + "zai-org/GLM-5.1", + "zai-org/GLM-5", + "zai-org/GLM-5-FP8", + "zai-org/GLM-5.1-FP8", + "zai-org/GLM-4.7-Flash", + "zai-org/GLM-4.6", + "zai-org/GLM-4.5", + "zai-org/GLM-4.5-Air", + # Open-weight mid-size frontier + "Qwen/Qwen3.6-27B", + "Qwen/Qwen3-32B", + "Qwen/Qwen3-14B", + "Qwen/Qwen3-8B", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Instruct", + "Qwen/Qwen3-235B-A22B", + "Qwen/Qwen3-4B-Instruct-2507", + # Reasoning/thinking lines that do not auto-surface via cardinality + "Qwen/QwQ-32B", + "Qwen/Qwen3-4B-Thinking-2507", + "deepseek-ai/DeepSeek-R1", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + # Other current open releases + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "google/gemma-3-27b-it", + "google/gemma-3-12b-it", + "google/gemma-4-31B-it", + "google/gemma-4-26B-A4B-it", + "meta-llama/Llama-3.3-70B-Instruct", + "meta-llama/Llama-4-Maverick-17B-128E-Instruct", + "meta-llama/Llama-4-Scout-17B-16E-Instruct", + "microsoft/phi-4", + "microsoft/Phi-4-mini-instruct", + "mistralai/Mistral-Large-Instruct-2411", + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "mistralai/Mistral-Small-3.1-24B-Instruct-2503", + "mistralai/Devstral-Small-2505", + "mistralai/Codestral-22B-v0.1", + "MiniMaxAI/MiniMax-M2", + "MiniMaxAI/MiniMax-M2.5", + # IBM Granite latest open releases + "ibm-granite/granite-4.0-h-small", + "ibm-granite/granite-4.0-h-tiny", + "ibm-granite/granite-3.3-8b-instruct", + "ibm-granite/granite-3.3-2b-instruct", + # AllenAI Olmo-3 + "allenai/Olmo-3-7B-Instruct", + "allenai/Olmo-3-1025-7B", + # Nemotron 3 series + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", +) + + +def _hf_api_url(path: str) -> str: + raw_endpoint = os.environ.get("HF_ENDPOINT") + endpoint = _DEFAULT_HF_ENDPOINT if raw_endpoint is None else raw_endpoint.strip() + if not endpoint: + raise ValueError("HF_ENDPOINT must not be empty") + if not endpoint.startswith(("http://", "https://")): + raise ValueError("HF_ENDPOINT must start with http:// or https://") + endpoint = endpoint.rstrip("/") + return f"{endpoint}/api/{path.lstrip('/')}" + + +def _model_list_params(limit: int, sort: str, filter_value: str | None = None) -> dict: + params = { + "pipeline_tag": "text-generation", + "sort": sort, + "limit": str(limit), + "expand[]": _MODEL_EXPANDS, + } + if filter_value: + params["filter"] = filter_value + return params + + +def _append_new_models( + data_list: list[dict], + models: list[ModelInfo], + seen_ids: set[str], +) -> None: + for data in data_list: + if data.get("id") not in seen_ids: + model = _parse_model(data) + if model: + models.append(model) + seen_ids.add(model.id) + + +async def _fetch_model_list( + client: httpx.AsyncClient, + params: dict, +) -> list[dict]: + resp = await get_with_retries(client, _hf_api_url("models"), params=params) + resp.raise_for_status() + return resp.json() + + +async def _fetch_frontier_models( + client: httpx.AsyncClient, + models: list[ModelInfo], + seen_ids: set[str], +) -> None: + for model_id in _FRONTIER_MODEL_IDS: + if model_id in seen_ids: + continue + try: + resp = await get_with_retries( + client, + _hf_api_url(f"models/{model_id}"), + params={"expand[]": _MODEL_DETAIL_EXPANDS}, + ) + if resp.status_code >= 400: + logger.debug( + f"Frontier fetch skipped {model_id}: HTTP {resp.status_code}" + ) + continue + data = resp.json() + except (httpx.HTTPError, ValueError) as e: + logger.debug(f"Frontier fetch failed for {model_id}: {e}") + continue + model = _parse_model(data) + if model: + models.append(model) + seen_ids.add(model.id) + + +async def fetch_models( + limit: int = 300, include_vision: bool = True +) -> list[ModelInfo]: + """Fetch popular models from HuggingFace Hub.""" + models: list[ModelInfo] = [] + + async with httpx.AsyncClient( + timeout=30.0, + follow_redirects=True, + headers={"Accept-Encoding": DEFAULT_ACCEPT_ENCODING}, + ) as client: + logger.debug(f"Fetching models from HF API (limit={limit})") + data_list = await _fetch_model_list( + client, _model_list_params(limit, sort="downloads") + ) + for data in data_list: + model = _parse_model(data) + if model: + models.append(model) + + logger.debug("Fetching GGUF models from HF API") + gguf_data_list = await _fetch_model_list( + client, _model_list_params(limit, sort="downloads", filter_value="gguf") + ) + + seen_ids = {m.id for m in models} + _append_new_models(gguf_data_list, models, seen_ids) + + logger.debug("Fetching recent GGUF models from HF API") + recent_data_list = await _fetch_model_list( + client, _model_list_params(limit, sort="lastModified", filter_value="gguf") + ) + _append_new_models(recent_data_list, models, seen_ids) + + for filter_value in (None, "gguf"): + logger.debug( + f"Fetching trending {filter_value or 'all'} models from HF API" + ) + try: + trending_data_list = await _fetch_model_list( + client, + _model_list_params( + limit, sort="trending", filter_value=filter_value + ), + ) + except (httpx.HTTPError, ValueError) as e: + logger.debug(f"Trending fetch skipped: {e}") + continue + _append_new_models(trending_data_list, models, seen_ids) + + await _fetch_frontier_models(client, models, seen_ids) + + if include_vision: + for pipeline_tag in ("image-text-to-text",): + mm_params = { + "pipeline_tag": pipeline_tag, + "sort": "downloads", + "limit": str(limit), + "expand[]": _MODEL_EXPANDS, + } + logger.debug(f"Fetching {pipeline_tag} models from HF API") + mm_data_list = await _fetch_model_list(client, mm_params) + _append_new_models(mm_data_list, models, seen_ids) + + logger.debug(f"Fetched {len(models)} models total") + return models + + +async def fetch_model_published_at(model_ids: list[str]) -> dict[str, str]: + """Fetch published timestamps for specific model IDs.""" + unique_ids = sorted({m for m in model_ids if m}) + if not unique_ids: + return {} + + async with httpx.AsyncClient( + timeout=20.0, + follow_redirects=True, + headers={"Accept-Encoding": DEFAULT_ACCEPT_ENCODING}, + ) as client: + tasks = [ + client.get( + _hf_api_url(f"models/{model_id}"), + params={"expand[]": ["createdAt", "lastModified"]}, + ) + for model_id in unique_ids + ] + responses = await asyncio.gather(*tasks, return_exceptions=True) + + result: dict[str, str] = {} + for model_id, resp in zip(unique_ids, responses, strict=False): + if isinstance(resp, Exception): + logger.debug("Failed to fetch model detail for %s: %s", model_id, resp) + continue + if resp.status_code >= 400: + logger.debug( + "Failed to fetch model detail for %s: HTTP %s", + model_id, + resp.status_code, + ) + continue + try: + data = resp.json() + except ValueError: + continue + published_at = _extract_published_at(data) + if published_at: + result[model_id] = published_at + return result diff --git a/src/whichllm/models/http.py b/src/whichllm/models/http.py new file mode 100644 index 0000000..a71b076 --- /dev/null +++ b/src/whichllm/models/http.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import asyncio +import random + +import httpx + +RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504} +DEFAULT_ACCEPT_ENCODING = "gzip, deflate" + + +async def get_with_retries( + client: httpx.AsyncClient, + url: str, + *, + attempts: int = 3, + base_delay: float = 0.25, + max_delay: float = 2.0, + jitter: float = 0.1, + retry_status_codes: set[int] | None = None, + **kwargs, +) -> httpx.Response: + """GET with bounded retry/backoff for transient HTTP failures.""" + retry_codes = retry_status_codes or RETRYABLE_STATUS_CODES + last_attempt = max(1, attempts) - 1 + + for attempt in range(last_attempt + 1): + try: + response = await client.get(url, **kwargs) + except (httpx.TimeoutException, httpx.TransportError): + if attempt >= last_attempt: + raise + else: + if response.status_code not in retry_codes or attempt >= last_attempt: + return response + + delay = min(max_delay, base_delay * (2**attempt)) + if jitter > 0: + delay += random.uniform(0, jitter) + if delay > 0: + await asyncio.sleep(delay) + + raise RuntimeError("unreachable retry state") diff --git a/src/whichllm/models/parameters.py b/src/whichllm/models/parameters.py new file mode 100644 index 0000000..9dd6938 --- /dev/null +++ b/src/whichllm/models/parameters.py @@ -0,0 +1,218 @@ +"""Parameter-count and MoE metadata normalization helpers.""" + +from __future__ import annotations + +import re + + +def _extract_size_hint_from_id(model_id: str | None) -> int | None: + """Extract parameter size hint (in params) from model ID like 27B or 30B-A3B.""" + if not model_id: + return None + lower = model_id.lower() + matches = re.findall(r"(\d+(?:\.\d+)?)b(?:-a\d+(?:\.\d+)?b)?", lower) + if not matches: + return None + try: + max_b = max(float(m) for m in matches) + except ValueError: + return None + if max_b <= 0: + return None + return int(max_b * 1e9) + + +def _extract_active_size_hint_from_id(model_id: str | None) -> int | None: + """Extract MoE active parameter hint from names like 35B-A3B.""" + if not model_id: + return None + lower = model_id.lower() + matches = re.findall(r"\d+(?:\.\d+)?b[-_]?a(\d+(?:\.\d+)?)b", lower) + if not matches: + return None + try: + max_b = max(float(m) for m in matches) + except ValueError: + return None + if max_b <= 0: + return None + return int(max_b * 1e9) + + +def _is_quantized_repo_name(model_id: str) -> bool: + """Detect quantized/non-base repository naming patterns.""" + lower = model_id.lower() + return bool(re.search(r"(gptq|awq|bnb|4bit|int4|int8|fp8|gguf|quant)", lower)) + + +def _lookup_curated_count(mapping: dict[str, int], model_id: str) -> int | None: + value = mapping.get(model_id) + if value is not None: + return value + + model_id_folded = model_id.casefold() + for key, value in mapping.items(): + if key.casefold() == model_id_folded: + return value + return None + + +def _resolve_moe_active_params( + total_params: int, + *model_refs: str | None, +) -> int | None: + """Resolve active params from curated data or A*B naming hints.""" + for ref in model_refs: + if not ref: + continue + active = _lookup_curated_count(_KNOWN_MOE_ACTIVE_PARAMS, ref) + if active and active > 0: + return active + + for ref in model_refs: + active = _extract_active_size_hint_from_id(ref) + if active and active > 0 and (total_params <= 0 or active < total_params): + return active + return None + + +def _normalize_param_count( + extracted: int, + model_id: str, + base_model: str | None, +) -> int: + """Normalize parameter count when metadata is inconsistent.""" + authoritative = _lookup_curated_count(_AUTHORITATIVE_PARAM_COUNTS, model_id) + if authoritative and authoritative > 0: + return authoritative + known = _lookup_curated_count(_KNOWN_PARAM_COUNTS, model_id) + if extracted <= 0: + return known or extracted + if known and extracted < int(known * 0.35): + return known + + hints = [ + h + for h in ( + _extract_size_hint_from_id(model_id), + _extract_size_hint_from_id(base_model), + ) + if h is not None + ] + if not hints: + return extracted + + hinted = max(hints) + if _is_quantized_repo_name(model_id): + if extracted < int(hinted * 0.70): + return hinted + elif extracted < int(hinted * 0.35): + return hinted + + return extracted + + +# Curated MoE active-parameter counts. Used when HF config lacks the +# `num_local_experts` / `num_experts_per_tok` keys that whichllm reads. +# Without this, frontier MoEs are scored as dense models which over-counts +# their VRAM cost and under-counts their inference speed. +_KNOWN_MOE_ACTIVE_PARAMS: dict[str, int] = { + "meta-llama/Llama-4-Scout-17B-16E-Instruct": 17_000_000_000, + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 17_000_000_000, + "Qwen/Qwen3-Next-80B-A3B-Instruct": 3_000_000_000, + "Qwen/Qwen3-30B-A3B": 3_000_000_000, + "Qwen/Qwen3-Coder-30B-A3B-Instruct": 3_000_000_000, + "Qwen/Qwen3-235B-A22B": 22_000_000_000, + "Qwen/Qwen3.5-397B-A17B": 17_000_000_000, + "deepseek-ai/DeepSeek-V3": 37_000_000_000, + "deepseek-ai/DeepSeek-V3-0324": 37_000_000_000, + "deepseek-ai/DeepSeek-V3.1": 37_000_000_000, + "deepseek-ai/DeepSeek-V3.2": 37_000_000_000, + "deepseek-ai/DeepSeek-V3.2-Exp": 37_000_000_000, + "deepseek-ai/DeepSeek-R1": 37_000_000_000, + "deepseek-ai/DeepSeek-R1-0528": 37_000_000_000, + "deepseek-ai/DeepSeek-V4-Pro": 49_000_000_000, + "deepseek-ai/DeepSeek-V4-Flash": 13_000_000_000, + "zai-org/GLM-4.5": 32_000_000_000, + "zai-org/GLM-4.5-Air": 12_000_000_000, + "zai-org/GLM-4.6": 32_000_000_000, + "zai-org/GLM-4.7": 32_000_000_000, + "zai-org/GLM-4.7-Flash": 12_000_000_000, + "zai-org/GLM-5": 40_000_000_000, + "zai-org/GLM-5-FP8": 40_000_000_000, + "zai-org/GLM-5.1": 40_000_000_000, + "zai-org/GLM-5.1-FP8": 40_000_000_000, + "moonshotai/Kimi-K2-Instruct": 32_000_000_000, + "moonshotai/Kimi-K2-Thinking": 32_000_000_000, + "MiniMaxAI/MiniMax-M2": 10_000_000_000, + "MiniMaxAI/MiniMax-M2.5": 10_000_000_000, + "XiaomiMiMo/MiMo-V2.5": 15_000_000_000, + "XiaomiMiMo/MiMo-V2.5-Pro": 42_000_000_000, + "XiaomiMiMo/MiMo-V2-Flash": 15_000_000_000, + "google/gemma-4-26B-A4B-it": 3_800_000_000, + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": 3_000_000_000, + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 3_000_000_000, + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": 12_000_000_000, + # OpenAI gpt-oss MoE family - 5B active for 20b/120b. + "openai/gpt-oss-20b": 3_600_000_000, + "openai/gpt-oss-120b": 5_100_000_000, +} + +# Hardcoded parameter counts for frontier models that HF's API leaves with +# missing safetensors/gguf/config metadata. Used as a last-resort fallback +# inside _extract_param_count so these models still enter the cache and become +# rankable. Maintain only entries that lack a size hint in the model ID itself. +_KNOWN_PARAM_COUNTS: dict[str, int] = { + "microsoft/phi-4": 14_700_000_000, + "microsoft/Phi-4-mini-instruct": 3_800_000_000, + "microsoft/Phi-4-multimodal-instruct": 5_600_000_000, + "microsoft/Phi-4-reasoning": 14_700_000_000, + "microsoft/Phi-4-reasoning-plus": 14_700_000_000, + "openai/gpt-oss-20b": 20_000_000_000, + "openai/gpt-oss-120b": 120_000_000_000, + # IBM Granite 4.0 family + "ibm-granite/granite-4.0-h-small": 32_000_000_000, + "ibm-granite/granite-4.0-h-tiny": 7_000_000_000, + "ibm-granite/granite-3.3-8b-instruct": 8_000_000_000, + "ibm-granite/granite-3.3-2b-instruct": 2_000_000_000, + # AllenAI Olmo-3 + "allenai/Olmo-3-7B-Instruct": 7_000_000_000, + "allenai/Olmo-3-1025-7B": 7_000_000_000, + # Llama 4 MoE totals: repo names advertise active size, but the total + # weight footprint is much larger. + "meta-llama/Llama-4-Scout-17B-16E-Instruct": 109_000_000_000, + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 400_000_000_000, + "deepseek-ai/DeepSeek-R1": 671_000_000_000, + "deepseek-ai/DeepSeek-R1-0528": 671_000_000_000, + "deepseek-ai/DeepSeek-V3": 671_000_000_000, + "deepseek-ai/DeepSeek-V3-0324": 671_000_000_000, + "deepseek-ai/DeepSeek-V3.1": 671_000_000_000, + "deepseek-ai/DeepSeek-V3.2": 685_000_000_000, + "deepseek-ai/DeepSeek-V4-Pro": 1_600_000_000_000, + "deepseek-ai/DeepSeek-V4-Flash": 284_000_000_000, + "moonshotai/Kimi-K2-Instruct": 1_026_000_000_000, + "moonshotai/Kimi-K2-Thinking": 1_026_000_000_000, + "XiaomiMiMo/MiMo-V2.5": 310_000_000_000, + "XiaomiMiMo/MiMo-V2.5-Pro": 1_020_000_000_000, + "XiaomiMiMo/MiMo-V2-Flash": 309_000_000_000, + "zai-org/GLM-4.5": 355_000_000_000, + "zai-org/GLM-4.5-Air": 106_000_000_000, + "zai-org/GLM-4.6": 355_000_000_000, + "zai-org/GLM-4.7": 355_000_000_000, + "zai-org/GLM-4.7-Flash": 30_000_000_000, + "zai-org/GLM-5": 744_000_000_000, + "zai-org/GLM-5-FP8": 744_000_000_000, + "zai-org/GLM-5.1": 744_000_000_000, + "zai-org/GLM-5.1-FP8": 744_000_000_000, + "MiniMaxAI/MiniMax-M2": 230_000_000_000, + "MiniMaxAI/MiniMax-M2.5": 230_000_000_000, + "stepfun-ai/Step-3.5-Flash": 30_000_000_000, +} + +# Curated counts that should win even when the HF API exposes safetensors +# metadata. Some mixed-precision MoEs publish compressed checkpoint tensor +# counts that understate the model-card capacity used for ranking and planning. +_AUTHORITATIVE_PARAM_COUNTS: dict[str, int] = { + "deepseek-ai/DeepSeek-V4-Pro": 1_600_000_000_000, + "deepseek-ai/DeepSeek-V4-Flash": 284_000_000_000, +} diff --git a/src/whichllm/models/parser.py b/src/whichllm/models/parser.py new file mode 100644 index 0000000..feb2dfc --- /dev/null +++ b/src/whichllm/models/parser.py @@ -0,0 +1,295 @@ +"""Parse HuggingFace API model payloads into ModelInfo objects.""" + +from __future__ import annotations + +import statistics + +from whichllm.models.gguf import _extract_gguf_variants +from whichllm.models.parameters import ( + _AUTHORITATIVE_PARAM_COUNTS, + _KNOWN_PARAM_COUNTS, + _extract_size_hint_from_id, + _lookup_curated_count, + _normalize_param_count, + _resolve_moe_active_params, +) +from whichllm.models.sliding_window import _resolve_sliding_window +from whichllm.models.types import ModelInfo + +_GENERAL_EVAL_KEYWORDS = ( + "mmlu", + "gpqa", + "gsm8k", + "hellaswag", + "arc", + "bbh", + "ifeval", + "truthfulqa", + "ceval", + "cmmlu", +) + + +def _extract_published_at(data: dict) -> str | None: + """Extract the best published timestamp candidate from an API response.""" + created = data.get("createdAt") + if isinstance(created, str) and created: + return created + modified = data.get("lastModified") + if isinstance(modified, str) and modified: + return modified + return None + + +def _normalize_eval_value(raw: object) -> float | None: + """Convert eval value to a comparable 0-100 score.""" + if not isinstance(raw, (int, float)): + return None + value = float(raw) + if value <= 0: + return None + if value <= 1.0: + value *= 100.0 + if value > 100.0: + return None + return value + + +def _is_general_eval_entry(entry: dict) -> bool: + """Keep eval entries that are broadly useful for general chat quality.""" + data = entry.get("data") + if not isinstance(data, dict): + return False + + notes = str(data.get("notes", "")).lower() + if "with tools" in notes: + return False + + dataset = data.get("dataset") + dataset_id = "" + task_id = "" + if isinstance(dataset, dict): + dataset_id = str(dataset.get("id", "")).lower() + task_id = str(dataset.get("task_id", "")).lower() + filename = str(entry.get("filename", "")).lower() + + return any( + k in dataset_id or k in task_id or k in filename for k in _GENERAL_EVAL_KEYWORDS + ) + + +def _extract_hf_eval_score(data: dict) -> float | None: + """Extract conservative aggregate score from HF evalResults.""" + eval_results = data.get("evalResults") + if not isinstance(eval_results, list) or not eval_results: + return None + + values: list[float] = [] + for entry in eval_results: + if not isinstance(entry, dict): + continue + if not _is_general_eval_entry(entry): + continue + data_obj = entry.get("data") + if not isinstance(data_obj, dict): + continue + normalized = _normalize_eval_value(data_obj.get("value")) + if normalized is not None: + values.append(normalized) + + if not values: + return None + return round(statistics.median(values), 1) + + +def _extract_param_count(model_data: dict) -> int: + """Extract parameter count from model data. + + Resolution order: + 1. authoritative model-card overrides for known mixed-precision MoEs + 2. safetensors metadata + 3. gguf metadata + 4. config estimate + 5. curated known counts + 6. name-based size hint + + Returns 0 if none of the above succeed. + """ + model_id = model_data.get("id", "") or "" + authoritative = _lookup_curated_count(_AUTHORITATIVE_PARAM_COUNTS, model_id) + if authoritative and authoritative > 0: + return authoritative + + safetensors = model_data.get("safetensors") + if safetensors and isinstance(safetensors, dict): + params = safetensors.get("total") + if params: + return int(params) + parameters = safetensors.get("parameters") + if isinstance(parameters, dict): + total = sum(parameters.values()) + if total > 0: + return total + + gguf_meta = model_data.get("gguf", {}) or {} + if isinstance(gguf_meta, dict): + total = gguf_meta.get("total") + if total and total > 0: + return int(total) + + config = model_data.get("config", {}) or {} + hidden = config.get("hidden_size", 0) + layers = config.get("num_hidden_layers", 0) + vocab = config.get("vocab_size", 0) + if hidden and layers and vocab: + return 12 * layers * hidden * hidden + vocab * hidden * 2 + + known = _lookup_curated_count(_KNOWN_PARAM_COUNTS, model_id) + if known and known > 0: + return known + name_hint = _extract_size_hint_from_id(model_id) + if name_hint and name_hint > 0: + return name_hint + + return 0 + + +def _extract_architecture(config: dict) -> str: + """Extract architecture string from config.""" + arch_list = config.get("architectures", []) + if arch_list: + arch = arch_list[0].lower() + for name in [ + "llama", + "qwen2", + "mistral", + "mixtral", + "gemma", + "phi", + "starcoder", + "command", + "deepseek", + ]: + if name in arch: + return name + return arch.replace("forcausallm", "").replace("forconditionalgeneration", "") + model_type = config.get("model_type", "") + return model_type.lower() + + +def _extract_base_model(card_data: dict) -> str | None: + base_model_raw = card_data.get("base_model") + if isinstance(base_model_raw, str): + return base_model_raw + if isinstance(base_model_raw, list) and base_model_raw: + return base_model_raw[0] + return None + + +def _resolve_active_params( + config: dict, + param_count: int, + model_id: str, + base_model: str | None, +) -> tuple[bool, int | None]: + num_experts = 0 + for k in ( + "num_local_experts", + "num_experts", + "n_routed_experts", + "moe_num_experts", + "num_moe_experts", + "n_local_experts", + ): + v = config.get(k, 0) + if isinstance(v, int) and v > num_experts: + num_experts = v + + experts_per_tok = 0 + for k in ( + "num_experts_per_tok", + "moe_topk", + "moe_top_k", + "num_experts_per_token", + "top_k", + ): + v = config.get(k, 0) + if isinstance(v, int) and v > experts_per_tok: + experts_per_tok = v + + known_moe_active = _resolve_moe_active_params(param_count, model_id, base_model) + is_moe = num_experts > 0 or known_moe_active is not None + active_params = None + if is_moe: + if known_moe_active is not None: + active_params = known_moe_active + elif num_experts > 0: + ept = experts_per_tok if experts_per_tok > 0 else 2 + active_ratio = ept / num_experts + expert_fraction = 0.6 + active_params = int( + param_count * (1 - expert_fraction + expert_fraction * active_ratio) + ) + return is_moe, active_params + + +def _parse_model(data: dict) -> ModelInfo | None: + """Parse HF API response into ModelInfo.""" + model_id = data.get("id", "") + if not model_id: + return None + + config = data.get("config", {}) or {} + card_data = data.get("cardData", {}) or {} + base_model = _extract_base_model(card_data) + + param_count = _extract_param_count(data) + param_count = _normalize_param_count(param_count, model_id, base_model) + if param_count == 0: + return None + + is_moe, active_params = _resolve_active_params( + config, param_count, model_id, base_model + ) + gguf_variants = _extract_gguf_variants(data, param_count) + + architecture = _extract_architecture(config) + gguf_meta = data.get("gguf", {}) or {} + if not architecture and isinstance(gguf_meta, dict): + architecture = gguf_meta.get("architecture", "") + + context_length = config.get("max_position_embeddings") or config.get( + "max_sequence_length" + ) + if not context_length and isinstance(gguf_meta, dict): + context_length = gguf_meta.get("context_length") + + gguf_arch = gguf_meta.get("architecture") if isinstance(gguf_meta, dict) else None + sliding_window, swa_global_ratio = _resolve_sliding_window( + config, model_id, gguf_arch + ) + + benchmark_scores: dict[str, float] = {} + eval_score = _extract_hf_eval_score(data) + if eval_score is not None: + benchmark_scores["hf_eval"] = eval_score + + return ModelInfo( + id=model_id, + family_id=model_id, + name=model_id.split("/")[-1], + parameter_count=param_count, + parameter_count_active=active_params, + architecture=architecture, + is_moe=is_moe, + context_length=context_length, + license=card_data.get("license"), + published_at=_extract_published_at(data), + downloads=data.get("downloads", 0), + likes=data.get("likes", 0), + gguf_variants=gguf_variants, + benchmark_scores=benchmark_scores, + base_model=base_model, + sliding_window=sliding_window, + sliding_window_global_ratio=swa_global_ratio, + ) diff --git a/src/whichllm/models/serialization.py b/src/whichllm/models/serialization.py new file mode 100644 index 0000000..dd4c103 --- /dev/null +++ b/src/whichllm/models/serialization.py @@ -0,0 +1,94 @@ +"""ModelInfo cache serialization helpers.""" + +from __future__ import annotations + +from whichllm.models.parameters import ( + _normalize_param_count, + _resolve_moe_active_params, +) +from whichllm.models.types import GGUFVariant, ModelInfo + + +def models_to_dicts(models: list[ModelInfo]) -> list[dict]: + """Serialize models to dicts for caching.""" + result = [] + for m in models: + result.append( + { + "id": m.id, + "family_id": m.family_id, + "name": m.name, + "parameter_count": m.parameter_count, + "parameter_count_active": m.parameter_count_active, + "architecture": m.architecture, + "is_moe": m.is_moe, + "context_length": m.context_length, + "license": m.license, + "published_at": m.published_at, + "downloads": m.downloads, + "likes": m.likes, + "gguf_variants": [ + { + "filename": v.filename, + "quant_type": v.quant_type, + "file_size_bytes": v.file_size_bytes, + } + for v in m.gguf_variants + ], + "benchmark_scores": m.benchmark_scores, + "base_model": m.base_model, + "sliding_window": m.sliding_window, + "sliding_window_global_ratio": m.sliding_window_global_ratio, + } + ) + return result + + +def dicts_to_models(data: list[dict]) -> list[ModelInfo]: + """Deserialize models from cached dicts.""" + models = [] + for d in data: + base_model = d.get("base_model") + param_count = _normalize_param_count( + d["parameter_count"], + d["id"], + base_model, + ) + active_params = _resolve_moe_active_params( + param_count, + d["id"], + base_model, + d.get("name"), + d.get("architecture"), + ) + if active_params is None: + active_params = d.get("parameter_count_active") + models.append( + ModelInfo( + id=d["id"], + family_id=d.get("family_id", d["id"]), + name=d["name"], + parameter_count=param_count, + parameter_count_active=active_params, + architecture=d.get("architecture", ""), + is_moe=d.get("is_moe", False) or active_params is not None, + context_length=d.get("context_length"), + license=d.get("license"), + published_at=d.get("published_at"), + downloads=d.get("downloads", 0), + likes=d.get("likes", 0), + gguf_variants=[ + GGUFVariant( + filename=v["filename"], + quant_type=v["quant_type"], + file_size_bytes=v["file_size_bytes"], + ) + for v in d.get("gguf_variants", []) + ], + benchmark_scores=d.get("benchmark_scores", {}), + base_model=base_model, + sliding_window=d.get("sliding_window"), + sliding_window_global_ratio=d.get("sliding_window_global_ratio"), + ) + ) + return models diff --git a/src/whichllm/models/sliding_window.py b/src/whichllm/models/sliding_window.py new file mode 100644 index 0000000..3c7094e --- /dev/null +++ b/src/whichllm/models/sliding_window.py @@ -0,0 +1,95 @@ +"""Sliding-window-attention metadata resolution.""" + +from __future__ import annotations + + +# Sliding-window-attention (SWA) registry. We only model SWA KV-cache savings +# for architectures whose mainline runtimes actually honor interleaved SWA +# (llama.cpp's ISWA path, MLX). Each entry is (default_window_tokens, +# global_layer_ratio) where the ratio is the fraction of layers that use full +# (global) attention. Models outside this allowlist keep full-context KV so +# estimates stay conservative. +_SWA_ARCH_DEFAULTS: dict[str, tuple[int, float]] = { + "gemma2": (4096, 0.5), + "gemma3": (1024, 1.0 / 6.0), + "gpt_oss": (128, 0.5), + "cohere2": (4096, 0.25), +} + +# Map the many spellings an arch string can take (HF model_type, the +# ForCausalLM/ForConditionalGeneration class prefix, and GGUF metadata) onto a +# canonical key in _SWA_ARCH_DEFAULTS. +_SWA_ARCH_ALIASES: dict[str, str] = { + "gemma2": "gemma2", + "gemma2_text": "gemma2", + "gemma3": "gemma3", + "gemma3_text": "gemma3", + "gpt_oss": "gpt_oss", + "gptoss": "gpt_oss", + "cohere2": "cohere2", +} + + +def _swa_key_from_arch(arch: str | None) -> str | None: + """Resolve an arch string (model_type / class / gguf metadata) to a key.""" + if not arch: + return None + arch = arch.lower() + if arch in _SWA_ARCH_ALIASES: + return _SWA_ARCH_ALIASES[arch] + stripped = arch.replace("forcausallm", "").replace("forconditionalgeneration", "") + if stripped in _SWA_ARCH_ALIASES: + return _SWA_ARCH_ALIASES[stripped] + return None + + +def _swa_arch_key(config: dict, model_id: str, gguf_arch: str | None) -> str | None: + """Identify the SWA architecture key for a model, or None if not honored. + + Relies on authoritative metadata only: raw HF config model_type / + architectures and GGUF metadata architecture. When none are present the + model is left unhonored (full-context estimate), since a false positive + would under-count VRAM. + """ + model_type = config.get("model_type") + key = _swa_key_from_arch(model_type if isinstance(model_type, str) else None) + if key: + return key + + arch_list = config.get("architectures") or [] + if arch_list and isinstance(arch_list[0], str): + key = _swa_key_from_arch(arch_list[0]) + if key: + return key + + return _swa_key_from_arch(gguf_arch) + + +def _resolve_sliding_window( + config: dict, model_id: str, gguf_arch: str | None = None +) -> tuple[int | None, float | None]: + """Resolve (sliding_window, global_ratio) for honored SWA architectures. + + Returns (None, None) for every model outside the allowlist so the KV + estimate stays at full context (conservative). + """ + if config.get("use_sliding_window") is False: + return None, None + + key = _swa_arch_key(config, model_id, gguf_arch) + if key is None: + return None, None + + default_window, default_ratio = _SWA_ARCH_DEFAULTS[key] + + window = config.get("sliding_window") + if not isinstance(window, int) or window <= 0: + window = default_window + + pattern = config.get("sliding_window_pattern") + if isinstance(pattern, int) and pattern > 0: + global_ratio = 1.0 / pattern + else: + global_ratio = default_ratio + + return window, global_ratio diff --git a/src/whichllm/models/types.py b/src/whichllm/models/types.py new file mode 100644 index 0000000..56abcba --- /dev/null +++ b/src/whichllm/models/types.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class GGUFVariant: + filename: str + quant_type: str # "Q4_K_M", "Q8_0" etc + file_size_bytes: int + + +@dataclass +class ModelInfo: + id: str # HF repo ID + family_id: str # grouping key + name: str + parameter_count: int # total parameters + parameter_count_active: int | None = None # MoE active params + architecture: str = "" # "llama", "qwen2", "mixtral" etc + is_moe: bool = False + context_length: int | None = None + license: str | None = None + published_at: str | None = None + downloads: int = 0 + likes: int = 0 + gguf_variants: list[GGUFVariant] = field(default_factory=list) + benchmark_scores: dict[str, float] = field(default_factory=dict) + base_model: str | None = None # cardData.base_model + # Sliding-window-attention KV-cache modeling. Only populated for + # architectures whose mainline runtimes actually honor interleaved SWA + # (Gemma-2/3, gpt-oss, Cohere2); left None otherwise so VRAM estimates + # stay conservative. sliding_window is the local-attention window in + # tokens; sliding_window_global_ratio is the fraction of layers that use + # full (global) attention (0.0 = pure SWA, 1.0 = fully dense). + sliding_window: int | None = None + sliding_window_global_ratio: float | None = None + + +@dataclass +class ModelFamily: + family_id: str + display_name: str + base_model: ModelInfo + variants: list[ModelInfo] = field(default_factory=list) + best_benchmark: dict[str, float] = field(default_factory=dict) diff --git a/src/whichllm/output/__init__.py b/src/whichllm/output/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/whichllm/output/_console.py b/src/whichllm/output/_console.py new file mode 100644 index 0000000..4afc1f4 --- /dev/null +++ b/src/whichllm/output/_console.py @@ -0,0 +1,11 @@ +"""Canonical Rich Console instance shared by every output surface. + +Tests patch the ``console`` attribute on this module to capture output +(e.g. ``whichllm.output._console.console = Console(file=buf, ...)``). +Surface modules look up the console via this module so the patch +propagates without each module holding its own binding. +""" + +from rich.console import Console + +console = Console() diff --git a/src/whichllm/output/display.py b/src/whichllm/output/display.py new file mode 100644 index 0000000..36be9d1 --- /dev/null +++ b/src/whichllm/output/display.py @@ -0,0 +1,40 @@ +"""Compatibility shim: per-surface output modules now live alongside this file. + +This module re-exports the public ``display_*`` functions so existing imports +(``from whichllm.output.display import display_ranking``) keep working. New +code should import from the specific submodule: + +- ``whichllm.output.ranking`` for ranking + hardware tables +- ``whichllm.output.plan`` for the plan command +- ``whichllm.output.upgrade`` for the upgrade comparison +- ``whichllm.output.json_output`` for machine-readable JSON output +- ``whichllm.output.formatting`` for shared byte/param/date/color helpers +- ``whichllm.output._console`` for the shared Rich ``Console`` instance + +The shared ``console`` symbol is re-exported here for read access. Code that +needs to *replace* the console (e.g. test capture) should set +``whichllm.output._console.console`` so every surface picks up the change. +""" + +from whichllm.output._console import console +from whichllm.output.json_output import ( + display_json, + display_plan_json, + display_upgrade_json, +) +from whichllm.output.markdown import display_markdown +from whichllm.output.plan import display_plan +from whichllm.output.ranking import display_hardware, display_ranking +from whichllm.output.upgrade import display_upgrade + +__all__ = [ + "console", + "display_hardware", + "display_json", + "display_markdown", + "display_plan", + "display_plan_json", + "display_ranking", + "display_upgrade", + "display_upgrade_json", +] diff --git a/src/whichllm/output/formatting.py b/src/whichllm/output/formatting.py new file mode 100644 index 0000000..1444bbb --- /dev/null +++ b/src/whichllm/output/formatting.py @@ -0,0 +1,111 @@ +"""Shared low-level helpers: byte/param/date formatters and color blending.""" + +from __future__ import annotations + +from datetime import datetime +from math import log10 + +from whichllm.engine.types import CompatibilityResult + + +def _format_bytes(b: int) -> str: + """Format bytes as human-readable string.""" + if b >= 1024**3: + return f"{b / 1024**3:.1f} GB" + elif b >= 1024**2: + return f"{b / 1024**2:.0f} MB" + return f"{b / 1024:.0f} KB" + + +def _format_params(count: int) -> str: + """Format parameter count.""" + if count >= 1e9: + return f"{count / 1e9:.1f}B" + elif count >= 1e6: + return f"{count / 1e6:.0f}M" + return str(count) + + +def _format_downloads(downloads: int) -> str: + """Format download count for compact table display.""" + if downloads >= 1_000_000: + return f"{downloads / 1_000_000:.1f}M" + if downloads >= 1_000: + return f"{downloads / 1_000:.1f}K" + return str(downloads) + + +def _format_published_at(value: str | None) -> str: + """Format published datetime into YYYY-MM-DD.""" + if not value: + return "—" + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + return dt.strftime("%Y-%m-%d") + except ValueError: + return value[:10] if len(value) >= 10 else value + + +def _format_speed(result: CompatibilityResult) -> str: + speed = result.estimated_tok_per_sec + if speed is None: + return "[grey50]N/A[/]" + base = f"{speed:.1f} tok/s" + if speed < 4.0: + style = "red" + elif speed < 10.0: + style = "yellow" + elif speed < 30.0: + style = "green" + else: + style = "bright_green" + + marker = "" + if result.speed_confidence == "low": + marker = " ?" + elif result.speed_confidence == "medium": + marker = " ~" + return f"[{style}]{base}{marker}[/{style}]" + + +def _parse_published_at(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _lerp_channel(a: int, b: int, t: float) -> int: + return int(a + (b - a) * t) + + +def _blend_hex(a: tuple[int, int, int], b: tuple[int, int, int], t: float) -> str: + t = max(0.0, min(1.0, t)) + r = _lerp_channel(a[0], b[0], t) + g = _lerp_channel(a[1], b[1], t) + bch = _lerp_channel(a[2], b[2], t) + return f"#{r:02x}{g:02x}{bch:02x}" + + +def _downloads_style(downloads: int, min_log: float, max_log: float) -> str: + if downloads <= 0: + return "grey50" + dlog = log10(max(downloads, 1)) + span = max(max_log - min_log, 1e-6) + t = (dlog - min_log) / span + return _blend_hex((145, 80, 80), (55, 190, 120), t) + + +def _published_style( + published: datetime | None, + oldest_ts: float | None, + newest_ts: float | None, +) -> str: + if published is None or oldest_ts is None or newest_ts is None: + return "grey50" + pts = published.timestamp() + span = max(newest_ts - oldest_ts, 1e-6) + t = (pts - oldest_ts) / span + return _blend_hex((190, 85, 85), (80, 190, 110), t) diff --git a/src/whichllm/output/json_output.py b/src/whichllm/output/json_output.py new file mode 100644 index 0000000..1c91d6c --- /dev/null +++ b/src/whichllm/output/json_output.py @@ -0,0 +1,210 @@ +"""Machine-readable JSON output for ranking, plan, and upgrade surfaces.""" + +from __future__ import annotations + +import json + +from whichllm.engine.quantization import effective_quant_type, estimate_weight_bytes +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo +from whichllm.output import _console +from whichllm.output.upgrade import _summarize_row + + +def display_json(results: list[CompatibilityResult], hardware: HardwareInfo) -> None: + """Output ranking results as JSON.""" + output = { + "hardware": { + "gpus": [ + { + "name": g.name, + "vendor": g.vendor, + "vram_bytes": g.vram_bytes, + "usable_vram_bytes": g.usable_vram_bytes, + "memory_bandwidth_gbps": g.memory_bandwidth_gbps, + "shared_memory": g.shared_memory, + } + for g in hardware.gpus + ], + "cpu": hardware.cpu_name, + "cpu_cores": hardware.cpu_cores, + "ram_bytes": hardware.ram_bytes, + "ram_budget_bytes": hardware.ram_budget_bytes, + "budget_notes": hardware.budget_notes, + "os": hardware.os, + }, + "models": [ + { + "rank": i, + "model_id": r.model.id, + "artifact_repo_id": r.artifact_model.id if r.artifact_model else None, + "artifact_filename": ( + r.artifact_variant.filename if r.artifact_variant else None + ), + "parameter_count": r.model.parameter_count, + "published_at": r.model.published_at, + "downloads": r.model.downloads, + "quant_type": effective_quant_type(r.model, r.gguf_variant), + "file_size_bytes": ( + r.gguf_variant.file_size_bytes + if r.gguf_variant + else estimate_weight_bytes(r.model, None) + ), + "vram_required_bytes": r.vram_required_bytes, + "vram_available_bytes": r.vram_available_bytes, + "uses_multi_gpu": r.uses_multi_gpu, + "multi_gpu_effective_vram_bytes": r.multi_gpu_effective_vram_bytes, + "estimated_tok_per_sec": r.estimated_tok_per_sec, + "speed_confidence": r.speed_confidence, + "speed_range_tok_per_sec": ( + list(r.speed_range_tok_per_sec) + if r.speed_range_tok_per_sec + else None + ), + "speed_notes": r.speed_notes, + "quality_score": round(r.quality_score, 2), + "benchmark_status": r.benchmark_status, + "benchmark_source": r.benchmark_source, + "benchmark_confidence": round(r.benchmark_confidence, 2), + "fit_type": r.fit_type, + "can_run": r.can_run, + "warnings": r.warnings, + "license": r.model.license, + } + for i, r in enumerate(results, 1) + ], + } + _console.console.print_json(json.dumps(output, ensure_ascii=False)) + + +def display_plan_json( + model: ModelInfo, + context_length: int, + target_quant: str, +) -> None: + """Output plan results as JSON.""" + from whichllm.constants import ( + GPU_BANDWIDTH, + QUANT_BYTES_PER_WEIGHT, + QUANT_QUALITY_PENALTY, + ) + from whichllm.engine.performance import estimate_tok_per_sec + from whichllm.engine.vram import estimate_vram + from whichllm.hardware.types import GPUInfo + + _GiB = 1024**3 + + quant_levels = ["Q2_K", "Q3_K_M", "Q4_K_M", "Q5_K_M", "Q6_K", "Q8_0", "F16"] + vram_by_quant = {} + for qt in quant_levels: + bpw = QUANT_BYTES_PER_WEIGHT.get(qt) + if bpw is None: + continue + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=qt, file_size_bytes=fake_size + ) + vram_bytes = estimate_vram(model, fake_variant, context_length) + vram_by_quant[qt] = { + "vram_bytes": vram_bytes, + "quality_loss": QUANT_QUALITY_PENALTY.get(qt, 0.0), + } + + target_vram = vram_by_quant.get(target_quant.upper(), {}).get("vram_bytes", 0) + if target_vram == 0: + bpw = QUANT_BYTES_PER_WEIGHT.get(target_quant.upper(), 0.5625) + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=target_quant, file_size_bytes=fake_size + ) + target_vram = estimate_vram(model, fake_variant, context_length) + + _PLAN_GPUS: list[tuple[str, int]] = [ + ("RTX 4060", 8), + ("RTX 3060", 12), + ("RTX 4070", 12), + ("RTX 4080", 16), + ("RTX 4090", 24), + ("RX 7900 XTX", 24), + ("RTX 5090", 32), + ("A100 40GB", 40), + ("L40S", 48), + ("A100 80GB", 80), + ("H100", 80), + ("H200", 141), + ] + + bpw = QUANT_BYTES_PER_WEIGHT.get(target_quant.upper(), 0.5625) + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=target_quant, file_size_bytes=fake_size + ) + + gpus = [] + for gpu_name, vram_gb in _PLAN_GPUS: + vram_bytes = int(vram_gb * _GiB) + bandwidth = GPU_BANDWIDTH.get(gpu_name) + gpu_info = GPUInfo( + name=gpu_name, + vendor="nvidia", + vram_bytes=vram_bytes, + memory_bandwidth_gbps=bandwidth, + ) + if vram_bytes >= target_vram: + fit_type = "full_gpu" + elif vram_bytes >= target_vram * 0.4: + fit_type = "partial_offload" + else: + fit_type = "too_small" + + speed = None + if fit_type != "too_small" and bandwidth: + speed = round( + estimate_tok_per_sec(model, fake_variant, gpu_info, fit_type), 1 + ) + + gpus.append( + { + "name": gpu_name, + "vram_gb": vram_gb, + "fit_type": fit_type, + "estimated_tok_per_sec": speed, + } + ) + + output = { + "model": { + "id": model.id, + "parameter_count": model.parameter_count, + "architecture": model.architecture, + "context_length": model.context_length, + "license": model.license, + }, + "target_quant": target_quant, + "context_length": context_length, + "vram_by_quant": vram_by_quant, + "gpu_compatibility": gpus, + } + _console.console.print_json(json.dumps(output, ensure_ascii=False)) + + +def display_upgrade_json( + current_hw: HardwareInfo, + current_results: list, + target_results: list[tuple[str, HardwareInfo, list]], +) -> None: + """Emit the upgrade comparison as JSON for scripting.""" + current_row = _summarize_row("Current", current_hw, current_results) + rows = [] + for name, hw, res in target_results: + row = _summarize_row(name, hw, res) + row["delta_quality"] = row["top_quality"] - current_row["top_quality"] + row["delta_tok_s"] = row["top_tok_s"] - current_row["top_tok_s"] + rows.append(row) + _console.console.print_json( + json.dumps( + {"current": current_row, "targets": rows}, + ensure_ascii=False, + ) + ) diff --git a/src/whichllm/output/markdown.py b/src/whichllm/output/markdown.py new file mode 100644 index 0000000..dec6a06 --- /dev/null +++ b/src/whichllm/output/markdown.py @@ -0,0 +1,154 @@ +"""GitHub-Flavored Markdown output for ranking results.""" + +from __future__ import annotations + +from whichllm.engine.quantization import effective_quant_type +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import HardwareInfo +from whichllm.output import _console +from whichllm.output.formatting import ( + _format_bytes, + _format_downloads, + _format_params, + _format_published_at, +) + + +def _escape_markdown_cell(value: object) -> str: + text = "" if value is None else str(value) + return text.replace("\\", "\\\\").replace("|", "\\|").replace("\n", "
") + + +def _format_markdown_speed(result: CompatibilityResult) -> str: + speed = result.estimated_tok_per_sec + if speed is None: + return "N/A" + marker = "" + if result.speed_confidence == "low": + marker = " ?" + elif result.speed_confidence == "medium": + marker = " ~" + return f"{speed:.1f} tok/s{marker}" + + +def _format_markdown_score(result: CompatibilityResult) -> str: + score = f"{result.quality_score:.1f}" + if result.benchmark_status == "none": + return f"{score} ?" + if result.benchmark_status == "self_reported": + return f"{score} !sr" + if result.benchmark_status == "estimated": + return f"{score} ~" + return score + + +def _format_markdown_fit(fit_type: str) -> str: + labels = { + "full_gpu": "Full GPU", + "partial_offload": "Partial", + "cpu_only": "CPU only", + } + return labels.get(fit_type, fit_type) + + +def _format_markdown_params(result: CompatibilityResult) -> str: + params = _format_params(result.model.parameter_count) + if result.model.is_moe and result.model.parameter_count_active: + params += f" ({_format_params(result.model.parameter_count_active)}a)" + return params + + +def _format_markdown_model(result: CompatibilityResult) -> str: + if not result.artifact_model: + return result.model.id + return f"[{result.model.id}](https://huggingface.co/{result.artifact_model.id})" + + +def _markdown_table(headers: list[str], rows: list[list[str]]) -> str: + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append( + "| " + " | ".join(_escape_markdown_cell(cell) for cell in row) + " |" + ) + return "\n".join(lines) + + +def _write_markdown(text: str) -> None: + _console.console.file.write(text + "\n") + _console.console.file.flush() + + +def display_markdown( + results: list[CompatibilityResult], + hardware: HardwareInfo, + *, + show_status: bool = False, + empty_message: str | None = None, +) -> None: + """Emit ranking results as a pasteable GitHub-Flavored Markdown table.""" + lines = ["## Recommended Models", ""] + + if not results: + lines.append(empty_message or "No compatible models found for your hardware.") + _write_markdown("\n".join(lines)) + return + + if show_status: + mem_label = "VRAM" if hardware.gpus else "RAM" + headers = [ + "#", + "Model", + "Params", + "Quant", + "Fit", + mem_label, + "Speed", + "Published", + "Score", + "License", + ] + rows = [ + [ + str(index), + _format_markdown_model(result), + _format_markdown_params(result), + effective_quant_type(result.model, result.gguf_variant), + _format_markdown_fit(result.fit_type), + _format_bytes(result.vram_required_bytes), + _format_markdown_speed(result), + _format_published_at(result.model.published_at), + _format_markdown_score(result), + result.model.license or "-", + ] + for index, result in enumerate(results, 1) + ] + else: + headers = [ + "#", + "Model", + "Params", + "Quant", + "Published", + "Downloads", + "Score", + "License", + ] + rows = [ + [ + str(index), + _format_markdown_model(result), + _format_markdown_params(result), + effective_quant_type(result.model, result.gguf_variant), + _format_published_at(result.model.published_at), + _format_downloads(result.model.downloads), + _format_markdown_score(result), + result.model.license or "-", + ] + for index, result in enumerate(results, 1) + ] + + lines.append(_markdown_table(headers, rows)) + _write_markdown("\n".join(lines)) diff --git a/src/whichllm/output/plan.py b/src/whichllm/output/plan.py new file mode 100644 index 0000000..5b7090c --- /dev/null +++ b/src/whichllm/output/plan.py @@ -0,0 +1,158 @@ +"""Plan-command Rich output.""" + +from __future__ import annotations + +from rich.panel import Panel +from rich.table import Table + +from whichllm.models.types import GGUFVariant, ModelInfo +from whichllm.output import _console +from whichllm.output.formatting import _format_bytes, _format_params + + +def display_plan( + model: ModelInfo, + context_length: int, + target_quant: str, +) -> None: + """Display hardware requirements for a specific model.""" + from whichllm.constants import ( + GPU_BANDWIDTH, + QUANT_BYTES_PER_WEIGHT, + QUANT_QUALITY_PENALTY, + ) + from whichllm.engine.performance import estimate_tok_per_sec + from whichllm.engine.vram import estimate_vram + from whichllm.hardware.types import GPUInfo + + _GiB = 1024**3 + + # -- Model info panel -- + params = _format_params(model.parameter_count) + active = "" + if model.is_moe and model.parameter_count_active: + active = f" ({_format_params(model.parameter_count_active)} active)" + ctx = str(model.context_length) if model.context_length else "unknown" + + lines = [ + f"[bold cyan]Model:[/] {model.id}", + f"[bold cyan]Params:[/] {params}{active} | Arch: {model.architecture} | Context: {ctx}", + ] + if model.license: + lines.append(f"[bold cyan]License:[/] {model.license}") + panel = Panel("\n".join(lines), title="[bold]Model Info[/]", border_style="cyan") + _console.console.print(panel) + + # -- VRAM requirements by quantization -- + quant_levels = ["Q2_K", "Q3_K_M", "Q4_K_M", "Q5_K_M", "Q6_K", "Q8_0", "F16"] + vram_table = Table( + title=f"VRAM Required (context: {context_length})", show_lines=True + ) + vram_table.add_column("Quant", style="bold", width=8) + vram_table.add_column("VRAM", justify="right", width=10) + vram_table.add_column("Quality Loss", justify="right", width=12) + + target_vram = 0 + for qt in quant_levels: + bpw = QUANT_BYTES_PER_WEIGHT.get(qt) + if bpw is None: + continue + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=qt, file_size_bytes=fake_size + ) + vram_bytes = estimate_vram(model, fake_variant, context_length) + penalty = QUANT_QUALITY_PENALTY.get(qt, 0.0) + penalty_str = f"-{penalty * 100:.0f}%" if penalty > 0 else "0%" + marker = " ★" if qt.upper() == target_quant.upper() else "" + style = "bold green" if qt.upper() == target_quant.upper() else "" + vram_table.add_row( + f"{qt}{marker}", _format_bytes(vram_bytes), penalty_str, style=style + ) + if qt.upper() == target_quant.upper(): + target_vram = vram_bytes + + _console.console.print(vram_table) + + if target_vram == 0: + bpw = QUANT_BYTES_PER_WEIGHT.get(target_quant.upper(), 0.5625) + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=target_quant, file_size_bytes=fake_size + ) + target_vram = estimate_vram(model, fake_variant, context_length) + + # -- GPU compatibility table -- + _PLAN_GPUS: list[tuple[str, int]] = [ + ("RTX 4060", 8), + ("RTX 3060", 12), + ("RTX 4070", 12), + ("RTX 4080", 16), + ("RTX 4090", 24), + ("RX 7900 XTX", 24), + ("RTX 5090", 32), + ("A100 40GB", 40), + ("L40S", 48), + ("A100 80GB", 80), + ("H100", 80), + ("H200", 141), + ] + + gpu_table = Table( + title=f"GPU Compatibility ({target_quant}, {_format_bytes(target_vram)} required)", + show_lines=True, + ) + gpu_table.add_column("GPU", style="bold", min_width=14) + gpu_table.add_column("VRAM", justify="right", width=8) + gpu_table.add_column("Fit", justify="center", width=12) + gpu_table.add_column("Est. Speed", justify="right", width=10) + + bpw = QUANT_BYTES_PER_WEIGHT.get(target_quant.upper(), 0.5625) + fake_size = int(model.parameter_count * bpw) + fake_variant = GGUFVariant( + filename="", quant_type=target_quant, file_size_bytes=fake_size + ) + + min_full_gpu = None + for gpu_name, vram_gb in _PLAN_GPUS: + vram_bytes = int(vram_gb * _GiB) + bandwidth = GPU_BANDWIDTH.get(gpu_name) + gpu_info = GPUInfo( + name=gpu_name, + vendor="nvidia", + vram_bytes=vram_bytes, + memory_bandwidth_gbps=bandwidth, + ) + + if vram_bytes >= target_vram: + fit = "[green]✓ Full GPU[/]" + fit_type = "full_gpu" + if min_full_gpu is None: + min_full_gpu = (gpu_name, vram_gb) + elif vram_bytes >= target_vram * 0.4: + fit = "[yellow]~ Partial[/]" + fit_type = "partial_offload" + else: + fit = "[red]✗ Too small[/]" + fit_type = None + + if fit_type and bandwidth: + speed = estimate_tok_per_sec(model, fake_variant, gpu_info, fit_type) + speed_str = f"{speed:.1f} tok/s" + else: + speed_str = "—" + + gpu_table.add_row(gpu_name, f"{vram_gb} GB", fit, speed_str) + + _console.console.print(gpu_table) + + if min_full_gpu: + _console.console.print( + f" [green]★[/] Minimum GPU for full offload: " + f"[bold]{min_full_gpu[0]}[/] ({min_full_gpu[1]} GB) at {target_quant}" + ) + else: + _console.console.print( + f" [yellow]Note:[/] No single GPU can fully load this model at {target_quant}. " + "Consider a lower quantization or multi-GPU setup." + ) diff --git a/src/whichllm/output/ranking.py b/src/whichllm/output/ranking.py new file mode 100644 index 0000000..50b1c0e --- /dev/null +++ b/src/whichllm/output/ranking.py @@ -0,0 +1,348 @@ +"""Ranking and hardware Rich output surfaces.""" + +from __future__ import annotations + +import re +from math import log10 + +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from whichllm.engine.quantization import effective_quant_type +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import HardwareInfo +from whichllm.output import _console +from whichllm.output.formatting import ( + _downloads_style, + _format_bytes, + _format_downloads, + _format_params, + _format_published_at, + _format_speed, + _parse_published_at, + _published_style, +) + + +def _detect_specializations(model_id: str) -> list[str]: + """Detect task-specialized model hints from repository name.""" + lower = model_id.lower() + tags: list[str] = [] + if re.search(r"(coder|codegen|starcoder|program|coding)", lower): + tags.append("coding") + if re.search(r"(^|[-_/])(vl|vision|multimodal|llava|image)([-_/]|$)", lower): + tags.append("vision") + if re.search(r"(^|[-_/])math([-_/]|$)", lower): + tags.append("math") + return tags + + +def _artifact_model_id(result: CompatibilityResult) -> str: + if result.artifact_model: + return result.artifact_model.id + return result.model.id + + +def _top_pick_confidence(results: list[CompatibilityResult]) -> tuple[str, str]: + """Return confidence level and explanation for top pick.""" + top = results[0] + gap = (top.quality_score - results[1].quality_score) if len(results) > 1 else 999.0 + notes: list[str] = [] + if top.fit_type == "partial_offload": + notes.append("partial offload") + elif top.fit_type == "cpu_only": + notes.append("CPU-only") + if top.speed_confidence == "low": + notes.append("low-confidence speed") + risk_note = f", {', '.join(notes)}" if notes else "" + + if top.benchmark_status == "none": + return "Low", f"no benchmark data, gap +{gap:.1f}{risk_note}" + if top.benchmark_status == "self_reported": + return ( + "Low", + f"uploader-reported benchmark only (unverified), gap +{gap:.1f}{risk_note}", + ) + if top.benchmark_status == "estimated": + if gap >= 2.0: + confidence = "Medium" + else: + confidence = "Low" + if top.speed_confidence == "low" and confidence == "Medium": + confidence = "Low" + return confidence, f"estimated benchmark, gap +{gap:.1f}{risk_note}" + if gap >= 2.5: + confidence = "High" + reason = f"direct benchmark, gap +{gap:.1f}{risk_note}" + elif gap >= 1.0: + confidence = "Medium" + reason = f"direct benchmark, gap +{gap:.1f}{risk_note}" + else: + confidence = "Low" + reason = f"direct benchmark but very close (+{gap:.1f}){risk_note}" + + # オフロード/CPU-only/低信頼speedの1位は実運用で不確実性が高いため信頼度を1段階下げる + if top.fit_type != "full_gpu" or top.speed_confidence == "low": + if confidence == "High": + confidence = "Medium" + elif confidence == "Medium": + confidence = "Low" + return confidence, reason + + +def display_hardware(hw: HardwareInfo) -> None: + """Display hardware information panel.""" + lines: list[str] = [] + + if hw.gpus: + for i, gpu in enumerate(hw.gpus): + if gpu.shared_memory: + vram = ( + f"{_format_bytes(gpu.vram_bytes)} shared" + if gpu.vram_bytes > 0 + else "shared memory" + ) + else: + vram = _format_bytes(gpu.vram_bytes) + if ( + gpu.usable_vram_bytes is not None + and gpu.usable_vram_bytes < gpu.vram_bytes + ): + vram += f" (budget {_format_bytes(gpu.usable_vram_bytes)})" + bw = ( + f"{gpu.memory_bandwidth_gbps:.0f} GB/s" + if gpu.memory_bandwidth_gbps + else "N/A" + ) + cc = ( + f"CC {gpu.compute_capability[0]}.{gpu.compute_capability[1]}" + if gpu.compute_capability + else "" + ) + extra = [] + if cc: + extra.append(cc) + if gpu.cuda_version: + extra.append(f"CUDA {gpu.cuda_version}") + if gpu.rocm_version: + extra.append(f"ROCm {gpu.rocm_version}") + extra_str = f" ({', '.join(extra)})" if extra else "" + lines.append( + f"[bold green]GPU {i}:[/] {gpu.name} — {vram}{extra_str} — BW: {bw}" + ) + else: + lines.append("[yellow]No GPU detected[/] — CPU-only mode") + + avx_flags = [] + if hw.has_avx2: + avx_flags.append("AVX2") + if hw.has_avx512: + avx_flags.append("AVX-512") + avx_str = f" ({', '.join(avx_flags)})" if avx_flags else "" + lines.append(f"[bold blue]CPU:[/] {hw.cpu_name} — {hw.cpu_cores} cores{avx_str}") + + ram = _format_bytes(hw.ram_bytes) + if hw.ram_budget_bytes is not None and hw.ram_budget_bytes < hw.ram_bytes: + ram += f" (budget {_format_bytes(hw.ram_budget_bytes)})" + lines.append(f"[bold blue]RAM:[/] {ram}") + lines.append(f"[bold blue]Disk free:[/] {_format_bytes(hw.disk_free_bytes)}") + lines.append(f"[bold blue]OS:[/] {hw.os}") + for note in hw.budget_notes: + lines.append(f"[dim]{note}[/dim]") + + panel = Panel("\n".join(lines), title="[bold]Hardware Info[/]", border_style="blue") + _console.console.print(panel) + + +def display_ranking( + results: list[CompatibilityResult], + *, + has_gpu: bool = True, + show_status: bool = False, + empty_message: str | None = None, +) -> None: + """Display ranked model table.""" + if not results: + _console.console.print( + f"[yellow]{empty_message or 'No compatible models found for your hardware.'}[/]" + ) + return + + mem_label = "VRAM" if has_gpu else "RAM" + + table = Table(title="Recommended Models", show_lines=True) + table.add_column("#", style="bold", width=3, justify="right") + table.add_column("Model", style="cyan", min_width=14, overflow="fold") + table.add_column("Quant", justify="center", width=6) + if show_status: + table.add_column(f"Fit / {mem_label}", justify="center", width=8) + table.add_column("Speed", justify="right", width=12) + table.add_column("Published", justify="center", width=10) + else: + table.add_column("Params", justify="right", width=6) + table.add_column("Published", justify="center", width=10) + table.add_column("Downloads", justify="right", width=9) + table.add_column("Score", justify="right", width=5) + + download_logs = [ + log10(max(r.model.downloads, 1)) for r in results if r.model.downloads > 0 + ] + min_download_log = min(download_logs) if download_logs else 0.0 + max_download_log = max(download_logs) if download_logs else 1.0 + published_dates = [_parse_published_at(r.model.published_at) for r in results] + published_valid = [d for d in published_dates if d is not None] + oldest_ts = min((d.timestamp() for d in published_valid), default=None) + newest_ts = max((d.timestamp() for d in published_valid), default=None) + + for i, r in enumerate(results, 1): + quant = effective_quant_type(r.model, r.gguf_variant) + vram_str = _format_bytes(r.vram_required_bytes) + speed_str = _format_speed(r) + + score_val = f"{r.quality_score:.1f}" + if r.benchmark_status == "none": + score_str = f"[red]{score_val} ?[/red]" + elif r.benchmark_status == "self_reported": + score_str = f"[bright_yellow]{score_val} !sr[/bright_yellow]" + elif r.benchmark_status == "estimated": + score_str = f"[yellow]{score_val} ~[/yellow]" + else: + score_str = f"[green]{score_val}[/green]" + + fit_style = { + "full_gpu": "[green]Full GPU[/]", + "partial_offload": "[yellow]Partial[/]", + "cpu_only": "[red]CPU only[/]", + } + fit_str = fit_style.get(r.fit_type, r.fit_type) + published_dt = _parse_published_at(r.model.published_at) + published_str = Text( + _format_published_at(r.model.published_at), + style=_published_style(published_dt, oldest_ts, newest_ts), + ) + downloads_str = Text( + _format_downloads(r.model.downloads), + style=_downloads_style( + r.model.downloads, min_download_log, max_download_log + ), + ) + + params_str = _format_params(r.model.parameter_count) + if r.model.is_moe and r.model.parameter_count_active: + params_str += f" ({_format_params(r.model.parameter_count_active)}a)" + + model_link = Text(r.model.id, style="cyan") + model_link.stylize(f"link https://huggingface.co/{_artifact_model_id(r)}") + if show_status: + model_link.append(f"\n{params_str}", style="dim") + + row_cells = [ + str(i), + model_link, + quant, + ] + if show_status: + row_cells.extend( + [f"{fit_str}\n[dim]{vram_str}[/dim]", speed_str, published_str] + ) + else: + row_cells.append(params_str) + row_cells.extend([published_str, downloads_str]) + row_cells.append(score_str) + table.add_row(*row_cells) + + _console.console.print(table) + + has_estimated = any(r.benchmark_status == "estimated" for r in results) + has_self = any(r.benchmark_status == "self_reported" for r in results) + has_none = any(r.benchmark_status == "none" for r in results) + if has_estimated or has_none or has_self: + parts = [] + if has_self: + parts.append( + "[bright_yellow]!sr[/bright_yellow] = uploader-reported only (unverified)" + ) + if has_estimated: + parts.append("[yellow]Estimated / ~[/yellow] = inferred from model line") + if has_none: + parts.append("[red]None / ?[/red] = no benchmark data") + _console.console.print(f" [dim]Score:[/dim] {', '.join(parts)}") + + if show_status: + has_speed_medium = any(r.speed_confidence == "medium" for r in results) + has_speed_low = any(r.speed_confidence == "low" for r in results) + if has_speed_medium or has_speed_low: + parts = [] + if has_speed_medium: + parts.append("[yellow]~[/yellow] = estimated tok/s range") + if has_speed_low: + parts.append("[red]?[/red] = low-confidence/backend-sensitive tok/s") + _console.console.print(f" [dim]Speed:[/dim] {', '.join(parts)}") + + has_direct = any(r.benchmark_status == "direct" for r in results) + if not has_direct: + _console.console.print( + " [red]No confirmed winner:[/] direct benchmark data is missing for current candidates." + ) + + confidence, reason = _top_pick_confidence(results) + confidence_style = { + "High": "green", + "Medium": "yellow", + "Low": "red", + }[confidence] + _console.console.print( + f" Top pick confidence: [{confidence_style}]{confidence}[/{confidence_style}] ({reason})" + ) + + from whichllm.models.benchmark_sources import BENCHMARK_SNAPSHOT + + _console.console.print( + f" [dim]Benchmark reference: {BENCHMARK_SNAPSHOT} curated snapshot; " + "live AA / LiveBench / Aider merged when reachable.[/dim]" + ) + + # 上位が僅差なら「断定しすぎない」ための注意を表示する + if len(results) >= 2: + gap = results[0].quality_score - results[1].quality_score + if gap < 1.5: + _console.console.print( + f" [yellow]Note:[/] Top candidates are very close (#{1} vs #{2}: {gap:.1f} pts)." + ) + + # 上位に根拠が弱い候補がある場合は目立つ注意を出す + weak_top = [ + idx + 1 for idx, r in enumerate(results[:3]) if r.benchmark_status != "direct" + ] + if weak_top: + joined = ", ".join(f"#{i}" for i in weak_top) + _console.console.print( + f" [yellow]Caution:[/] Weaker benchmark evidence in top ranks: {joined}" + ) + + weak_speed_top = [ + idx + 1 for idx, r in enumerate(results[:3]) if r.speed_confidence == "low" + ] + if weak_speed_top: + joined = ", ".join(f"#{i}" for i in weak_speed_top) + _console.console.print( + f" [yellow]Speed caution:[/] Low-confidence speed estimates in top ranks: {joined}" + ) + + specialized: list[str] = [] + for idx, r in enumerate(results[:10], 1): + tags = _detect_specializations(r.model.id) + if tags: + joined_tags = "/".join(tags) + specialized.append(f"#{idx} {joined_tags}") + if specialized: + _console.console.print( + " [yellow]Task hint:[/] Specialized models detected in ranking: " + + ", ".join(specialized) + ) + + for i, r in enumerate(results[:3], 1): + if r.warnings: + for w in r.warnings: + _console.console.print(f" [yellow]Warning #{i} {r.model.name}:[/] {w}") diff --git a/src/whichllm/output/upgrade.py b/src/whichllm/output/upgrade.py new file mode 100644 index 0000000..6b51f47 --- /dev/null +++ b/src/whichllm/output/upgrade.py @@ -0,0 +1,126 @@ +"""Upgrade-command Rich output.""" + +from __future__ import annotations + +from rich.table import Table + +from whichllm.engine.quantization import effective_quant_type +from whichllm.hardware.types import HardwareInfo +from whichllm.output import _console + + +def _summarize_row(name: str, hw: HardwareInfo, results: list) -> dict: + """Reduce a (hardware, ranking) pair to one row for the upgrade table.""" + gpu_label = "CPU-only" + vram_gb = 0.0 + if hw.gpus: + g = max(hw.gpus, key=lambda x: x.vram_bytes) + gpu_label = g.name + vram_gb = g.vram_bytes / 1024**3 + if not results: + return { + "name": name, + "gpu": gpu_label, + "vram_gb": vram_gb, + "top_model": "—", + "top_quality": 0.0, + "top_tok_s": 0.0, + "top_speed_confidence": "low", + "top_speed_range_tok_per_sec": None, + "top_fit": "—", + "top_quant": "—", + } + r = results[0] + return { + "name": name, + "gpu": gpu_label, + "vram_gb": vram_gb, + "top_model": r.model.id, + "top_quality": float(r.quality_score), + "top_tok_s": float(r.estimated_tok_per_sec), + "top_speed_confidence": r.speed_confidence, + "top_speed_range_tok_per_sec": ( + list(r.speed_range_tok_per_sec) if r.speed_range_tok_per_sec else None + ), + "top_fit": r.fit_type, + "top_quant": ( + r.gguf_variant.quant_type + if r.gguf_variant + else effective_quant_type(r.model, None) + ), + } + + +def _upgrade_verdict(delta_q: float, delta_speed: float) -> str: + """Return a short verdict for an upgrade row.""" + if delta_q >= 12 and delta_speed >= 10: + return "[bold green]worth it[/]" + if delta_q >= 8 or delta_speed >= 20: + return "[green]meaningful[/]" + if delta_q >= 3 or delta_speed >= 5: + return "[yellow]marginal[/]" + if delta_q <= -3 or delta_speed <= -5: + return "[red]downgrade[/]" + return "[dim]flat[/]" + + +def display_upgrade( + current_hw: HardwareInfo, + current_results: list, + target_results: list[tuple[str, HardwareInfo, list]], +) -> None: + """Render the GPU-upgrade comparison table.""" + current_row = _summarize_row("Current", current_hw, current_results) + target_rows = [_summarize_row(name, hw, res) for name, hw, res in target_results] + + table = Table( + title="GPU upgrade comparison", + show_lines=False, + header_style="bold cyan", + ) + table.add_column("Setup", style="bold") + table.add_column("GPU", overflow="fold") + table.add_column("VRAM", justify="right") + table.add_column("Best model", overflow="fold") + table.add_column("Quant") + table.add_column("Quality", justify="right") + table.add_column("tok/s", justify="right") + table.add_column("ΔQ", justify="right") + table.add_column("Δtok/s", justify="right") + table.add_column("Verdict") + + table.add_row( + current_row["name"], + current_row["gpu"], + f"{current_row['vram_gb']:.0f} GB" + if current_row["vram_gb"] is not None + else "—", + current_row["top_model"], + current_row["top_quant"], + f"{current_row['top_quality']:.1f}", + f"{current_row['top_tok_s']:.0f}", + "—", + "—", + "—", + ) + for row in target_rows: + dq = row["top_quality"] - current_row["top_quality"] + ds = row["top_tok_s"] - current_row["top_tok_s"] + table.add_row( + row["name"], + row["gpu"], + f"{row['vram_gb']:.0f} GB" if row["vram_gb"] is not None else "—", + row["top_model"], + row["top_quant"], + f"{row['top_quality']:.1f}", + f"{row['top_tok_s']:.0f}", + f"{dq:+.1f}", + f"{ds:+.0f}", + _upgrade_verdict(dq, ds), + ) + + _console.console.print(table) + _console.console.print( + "[dim]Verdict: worth it (≥12pt Q & ≥10 tok/s lift) · meaningful (≥8pt Q or " + "≥20 tok/s) · marginal · flat (no change) · downgrade.[/]" + ) diff --git a/src/whichllm/utils.py b/src/whichllm/utils.py new file mode 100644 index 0000000..ca0dbc0 --- /dev/null +++ b/src/whichllm/utils.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import os +import re +from importlib.metadata import version, PackageNotFoundError +from pathlib import Path + +import click + + +def _current_version() -> str: + """Return installed package version.""" + try: + return version("whichllm") + except PackageNotFoundError: + return "unknown" + + +_SHORTHAND_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*([kmb])$", re.IGNORECASE) +_MULTIPLIERS = {"k": 1024, "m": 1024 * 1024, "b": 1024 * 1024 * 1024} + + +def parse_context_length(value: str) -> int: + """Parse a context length string, supporting shorthand like 64k or 128K. + + Accepts plain integers (e.g. "4096") or shorthand with a suffix: + k/K = x1,024 (64k -> 65536) + m/M = x1,048,576 + b/B = x1,073,741,824 + + Returns the integer context length. Raises ValueError on bad input. + """ + value = value.strip() + match = _SHORTHAND_RE.match(value) + if match: + number = float(match.group(1)) + suffix = match.group(2).lower() + result = int(number * _MULTIPLIERS[suffix]) + if result <= 0: + raise ValueError(f"Context length must be positive, got {value!r}") + return result + try: + result = int(value) + except ValueError: + raise ValueError( + f"Invalid context length {value!r}. " + "Use a plain integer (4096) or shorthand (64k, 128k)." + ) + if result <= 0: + raise ValueError(f"Context length must be positive, got {result}") + return result + + +class ContextLengthType(click.ParamType): + """Click parameter type that accepts integers or shorthand like 64k.""" + + name = "context_length" + + def convert(self, value, param, ctx): + if isinstance(value, int): + return value + try: + return parse_context_length(str(value)) + except ValueError as e: + self.fail(str(e), param, ctx) + + +CONTEXT_LENGTH = ContextLengthType() + + +def _cache_dir() -> Path: + """Return the whichllm cache directory, respecting XDG_CACHE_HOME.""" + base = os.environ.get("XDG_CACHE_HOME") + if base and Path(base).is_absolute(): + return Path(base) / "whichllm" + return Path.home() / ".cache" / "whichllm" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_aa_index.py b/tests/test_aa_index.py new file mode 100644 index 0000000..5a62ebf --- /dev/null +++ b/tests/test_aa_index.py @@ -0,0 +1,137 @@ +"""Tests for the Artificial Analysis Intelligence Index source. + +These cover the Next.js App Router (RSC) scraper that replaced the old +``__NEXT_DATA__`` extraction, the variant-stripping name canonicalization, +and the merge-over-curated-fallback behaviour of ``fetch_aa_index_scores``. +All tests are offline — network is served from an ``httpx.MockTransport``. +""" + +from __future__ import annotations + +import asyncio +import json + +import httpx +import pytest + +from whichllm.models.benchmark_sources.aa_index import ( + AA_LEADERBOARD_URL, + _canonical_name, + _decode_rsc_blob, + _extract_aa_pairs_from_html, + _normalize_aa_index, + fetch_aa_index_scores, + get_aa_curated_fallback, +) +from whichllm.models.benchmark_sources.types import ExtractionFailed + + +def _rsc_page(records: list[dict]) -> str: + """Build a minimal HTML page that embeds ``records`` the way the live + artificialanalysis.ai App Router page does: as a JSON-string-escaped + fragment inside ``self.__next_f.push([n, "..."])``.""" + # The fragment is an arbitrary slice of the RSC stream; the scraper only + # cares that it contains the "name"/"intelligenceIndex" key pairs. + fragment = ",".join( + '{"slug":"x","name":%s,"reasoningModel":false,' + '"intelligenceIndex":%s,"codingIndex":1.0}' + % (json.dumps(r["name"]), r["index"]) + for r in records + ) + chunk = json.dumps("3:[" + fragment + "]\n") + return ( + "" + "" + f"" + "" + ) + + +def test_canonical_name_strips_variants_and_separators(): + assert _canonical_name("Qwen3 14B (Reasoning)") == "qwen3 14b" + assert _canonical_name("Qwen3-14B") == "qwen3 14b" + # Separators normalize to single spaces (the table side is canonicalized + # the same way, so "GLM-5" and "GLM 5" still collide). + assert _canonical_name("GLM-5 (Non-reasoning)") == "glm 5" + assert _canonical_name("DeepSeek V4 Pro (Reasoning, Max Effort)") == ( + "deepseek v4 pro" + ) + + +def test_decode_rsc_blob_unescapes_chunks(): + page = _rsc_page([{"name": "Qwen3 14B (Reasoning)", "index": 33.0}]) + blob = _decode_rsc_blob(page) + assert '"name":"Qwen3 14B (Reasoning)"' in blob + assert '"intelligenceIndex":33.0' in blob + + +def test_extract_pairs_from_rsc_html(): + page = _rsc_page( + [ + {"name": "Qwen3 14B (Reasoning)", "index": 33.0}, + {"name": "Qwen3 14B (Non-reasoning)", "index": 30.0}, + {"name": "GLM-5 (Reasoning)", "index": 50.0}, + ] + ) + pairs = dict(_extract_aa_pairs_from_html(page)) + assert pairs["Qwen3 14B (Reasoning)"] == 33.0 + assert pairs["GLM-5 (Reasoning)"] == 50.0 + # The bounded regex must not leak one record's name into another's index. + assert len(pairs) == 3 + + +def test_extract_pairs_returns_empty_on_legacy_or_garbage_html(): + assert _extract_aa_pairs_from_html("no rsc here") == [] + + +def _run_fetch(html: str) -> dict[str, float]: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == AA_LEADERBOARD_URL + return httpx.Response(200, text=html) + + async def go() -> dict[str, float]: + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + return await fetch_aa_index_scores(client) + + return asyncio.run(go()) + + +def test_fetch_maps_canonical_names_and_merges_over_fallback(): + # "Qwen3 14B (Reasoning)" canonicalizes onto the "Qwen3 14B" table entry + # -> Qwen/Qwen3-14B, and a high live value must override the snapshot. + page = _rsc_page([{"name": "Qwen3 14B (Reasoning)", "index": 55.0}]) + scores = _run_fetch(page) + + fallback = get_aa_curated_fallback() + # Coverage never shrinks below the curated snapshot ... + assert set(fallback).issubset(set(scores)) + # ... and the live number wins where it is higher. + assert scores["Qwen/Qwen3-14B"] > fallback["Qwen/Qwen3-14B"] + + +def test_fetch_raises_when_no_records_found(): + with pytest.raises(ExtractionFailed): + _run_fetch("nothing to see") + + +def test_live_normalization_anchors_on_reworked_scale(): + # Retuned bounds keep the calibration: the top mapped open model lands ~95 + # and an 8B-class model lands ~40, on AA's reworked (compressed) raw scale. + assert _normalize_aa_index(44.3) == pytest.approx(95, abs=0.5) # top open model + assert _normalize_aa_index(7.4) == pytest.approx(40, abs=0.5) # 8B-class + # Values below the floor clamp at 0 (raw is always positive in practice). + assert _normalize_aa_index(-100.0) == 0.0 + assert _normalize_aa_index(60.0) == 100.0 + + +def test_curated_fallback_normalizes_refreshed_snapshot(): + # The snapshot holds refreshed raw AA values; get_aa_curated_fallback maps + # them onto the 0-100 scale with the retuned bounds. + fb = get_aa_curated_fallback() + assert fb["deepseek-ai/DeepSeek-V4-Pro"] == pytest.approx(95, abs=0.5) + assert fb["Qwen/Qwen3-8B"] == 40.0 + assert fb["XiaomiMiMo/MiMo-V2.5-Pro"] == pytest.approx(92, abs=0.5) + # Reworked scale ranks the strong 8B above the small/old peers. + assert fb["Qwen/Qwen3-8B"] > fb["Qwen/Qwen3-0.6B"] + assert all(0.0 < v <= 100.0 for v in fb.values()) diff --git a/tests/test_amd_detection.py b/tests/test_amd_detection.py new file mode 100644 index 0000000..f5d528c --- /dev/null +++ b/tests/test_amd_detection.py @@ -0,0 +1,302 @@ +"""Tests for AMD GPU detection fallbacks.""" + +from __future__ import annotations + +import subprocess +from io import StringIO + +from rich.console import Console + +from whichllm.hardware import amd +from whichllm.hardware.types import GPUInfo, HardwareInfo + + +def test_detect_amd_gpu_from_lspci_when_rocm_smi_missing(monkeypatch): + output = ( + 'c1:00.0 "VGA compatible controller" "Advanced Micro Devices, Inc. ' + '[AMD/ATI]" "Strix Halo [Radeon 8060S]" -r00 "Framework" "Device 0001"\n' + ) + + def fake_run(args, **kwargs): + if args[0] == "rocm-smi": + raise FileNotFoundError + return subprocess.CompletedProcess(args, 0, stdout=output, stderr="") + + monkeypatch.setattr(amd.subprocess, "run", fake_run) + + gpus = amd.detect_amd_gpus() + + assert len(gpus) == 1 + assert gpus[0].vendor == "amd" + assert gpus[0].vram_bytes == 0 + assert gpus[0].shared_memory is True + assert gpus[0].memory_bandwidth_gbps == 256.0 + assert "Radeon 8060S" in gpus[0].name + + +def test_detect_strix_halo_rocm_smi_does_not_treat_aperture_as_vram(monkeypatch): + def fake_run(args, **kwargs): + if args[:2] == ["rocm-smi", "--showproductname"]: + return subprocess.CompletedProcess( + args, + 0, + stdout='{"card0": {"Card SKU": "STRXLGEN"}}', + stderr="", + ) + if args[:3] == ["rocm-smi", "--showmeminfo", "vram"]: + return subprocess.CompletedProcess( + args, + 0, + stdout='{"card0": {"VRAM Total Memory (B)": "536870912"}}', + stderr="", + ) + if args[:2] == ["rocm-smi", "--showdriverversion"]: + return subprocess.CompletedProcess( + args, + 0, + stdout='{"card0": {"Driver version": "7.0.3"}}', + stderr="", + ) + raise AssertionError(args) + + monkeypatch.setattr(amd.subprocess, "run", fake_run) + + gpus = amd.detect_amd_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "STRXLGEN" + assert gpus[0].vendor == "amd" + assert gpus[0].shared_memory is True + assert gpus[0].vram_bytes == 0 + assert gpus[0].rocm_version == "7.0.3" + assert gpus[0].memory_bandwidth_gbps == 256.0 + + +def test_detect_amd_gpu_ignores_intel_only_lspci(monkeypatch): + """Regression: an Intel VGA row must not be reported as AMD just + because 'Intel Corporation' contains the substring 'ati'.""" + output = ( + '00:02.0 "VGA compatible controller" "Intel Corporation" ' + '"Alder Lake-P GT1 [UHD Graphics]" -r0c -p00 ' + '"IP3 Tech (HK) Limited" "Device 8027"\n' + ) + + def fake_run(args, **kwargs): + if args[0] == "rocm-smi": + raise FileNotFoundError + return subprocess.CompletedProcess(args, 0, stdout=output, stderr="") + + monkeypatch.setattr(amd.subprocess, "run", fake_run) + # Isolate the lspci path: don't let a real sysfs probe leak in. + monkeypatch.setattr(amd, "_detect_from_sysfs", lambda: []) + + assert amd.detect_amd_gpus() == [] + + +def test_detect_amd_gpu_from_sysfs_when_lspci_missing(monkeypatch, tmp_path): + card = tmp_path / "card0" / "device" + card.mkdir(parents=True) + (card / "vendor").write_text("0x1002\n") + (card / "product_name").write_text("AMD Radeon RX 9060 XT\n") + (card / "mem_info_vram_total").write_text(str(16 * 1024**3)) + + monkeypatch.setattr(amd, "_detect_from_lspci", lambda: []) + original_sysfs = amd._detect_from_sysfs + monkeypatch.setattr(amd, "_detect_from_sysfs", lambda: original_sysfs(tmp_path)) + + gpus = amd._detect_amd_gpus_fallback() + + assert len(gpus) == 1 + assert gpus[0].vendor == "amd" + assert gpus[0].name == "AMD Radeon RX 9060 XT" + assert gpus[0].vram_bytes == 16 * 1024**3 + assert gpus[0].shared_memory is False + + +def test_display_amd_shared_memory_without_zero_kb(monkeypatch): + from whichllm.output import _console as console_mod + from whichllm.output import display as display_mod + + buf = StringIO() + monkeypatch.setattr(console_mod, "console", Console(file=buf, force_terminal=False)) + + display_mod.display_hardware( + HardwareInfo( + gpus=[ + GPUInfo( + name="Strix Halo [Radeon 8060S]", + vendor="amd", + vram_bytes=0, + shared_memory=True, + ) + ], + cpu_name="CPU", + cpu_cores=16, + ram_bytes=128 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + ) + + output = buf.getvalue() + assert "shared memory" in output + assert "256 GB/s" not in output + assert "0 KB" not in output + + +# ---------- Issue #61: RX 6750 XT detection ---------- + + +def test_sysfs_generic_name_enriched_by_lspci(monkeypatch, tmp_path): + """When sysfs gives 'AMD Graphics' and lspci gives a descriptive name, + the fallback should use the lspci name with sysfs VRAM.""" + _GiB = 1024**3 + + # sysfs: generic name but has VRAM + card = tmp_path / "card0" / "device" + card.mkdir(parents=True) + (card / "vendor").write_text("0x1002\n") + (card / "mem_info_vram_total").write_text(str(12 * _GiB)) + # no product_name → falls back to "AMD Graphics" + + lspci_name = "Navi 22 [Radeon RX 6700/6700 XT/6750 XT / 6800M/6850M XT]" + original_sysfs = amd._detect_from_sysfs + monkeypatch.setattr(amd, "_detect_from_sysfs", lambda: original_sysfs(tmp_path)) + monkeypatch.setattr(amd, "_detect_from_lspci", lambda: [lspci_name]) + + gpus = amd._detect_amd_gpus_fallback() + + assert len(gpus) == 1 + assert gpus[0].name == lspci_name + assert gpus[0].vram_bytes == 12 * _GiB + assert gpus[0].shared_memory is False + + +def test_sysfs_product_name_preferred_over_lspci(monkeypatch, tmp_path): + """When sysfs gives a real product name, it should be used even if + lspci is also available.""" + _GiB = 1024**3 + + card = tmp_path / "card0" / "device" + card.mkdir(parents=True) + (card / "vendor").write_text("0x1002\n") + (card / "product_name").write_text("AMD Radeon RX 6750 XT\n") + (card / "mem_info_vram_total").write_text(str(12 * _GiB)) + + original_sysfs = amd._detect_from_sysfs + monkeypatch.setattr(amd, "_detect_from_sysfs", lambda: original_sysfs(tmp_path)) + monkeypatch.setattr( + amd, + "_detect_from_lspci", + lambda: ["Navi 22 [Radeon RX 6700/6700 XT/6750 XT / 6800M/6850M XT]"], + ) + + gpus = amd._detect_amd_gpus_fallback() + + assert len(gpus) == 1 + assert gpus[0].name == "AMD Radeon RX 6750 XT" + assert gpus[0].vram_bytes == 12 * _GiB + assert gpus[0].memory_bandwidth_gbps == 432.0 + + +def test_lspci_enriched_with_sysfs_vram_when_sysfs_detection_fails( + monkeypatch, tmp_path +): + """When _detect_from_sysfs returns nothing but _read_sysfs_amd_vram + succeeds, lspci names should still get VRAM data.""" + _GiB = 1024**3 + + # _detect_from_sysfs returns nothing (e.g. product_name absent AND + # the card dir structure confuses the glob), but individual VRAM reads + # via _read_sysfs_amd_vram still work. + monkeypatch.setattr(amd, "_detect_from_sysfs", lambda: []) + monkeypatch.setattr( + amd, + "_detect_from_lspci", + lambda: ["Navi 22 [Radeon RX 6700/6700 XT/6750 XT / 6800M/6850M XT]"], + ) + monkeypatch.setattr(amd, "_read_sysfs_amd_vram", lambda: [12 * _GiB]) + + gpus = amd._detect_amd_gpus_fallback() + + assert len(gpus) == 1 + assert gpus[0].vram_bytes == 12 * _GiB + assert gpus[0].shared_memory is False + + +def test_lookup_bandwidth_compound_lspci_name(): + """The bandwidth lookup should resolve compound lspci names by + splitting on '/' and re-applying the 'RX ' prefix.""" + # Direct substring match works for clean names + assert amd._lookup_bandwidth("AMD Radeon RX 6750 XT") == 432.0 + assert amd._lookup_bandwidth("AMD Radeon RX 6700 XT") == 384.0 + + # Compound lspci name — first matching segment wins + compound = "Navi 22 [Radeon RX 6700/6700 XT/6750 XT / 6800M/6850M XT]" + bw = amd._lookup_bandwidth(compound) + assert bw is not None + assert bw > 0 + + +def test_display_amd_dgpu_does_not_say_shared_memory(monkeypatch): + """A discrete AMD GPU with VRAM must NOT display 'shared memory'.""" + from whichllm.output import _console as console_mod + from whichllm.output import display as display_mod + + buf = StringIO() + monkeypatch.setattr(console_mod, "console", Console(file=buf, force_terminal=False)) + + display_mod.display_hardware( + HardwareInfo( + gpus=[ + GPUInfo( + name="AMD Radeon RX 6750 XT", + vendor="amd", + vram_bytes=12 * 1024**3, + memory_bandwidth_gbps=432.0, + shared_memory=False, + ) + ], + cpu_name="AMD Ryzen 9 5900X", + cpu_cores=12, + ram_bytes=128 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + ) + + output = buf.getvalue() + assert "12.0 GB" in output + assert "432 GB/s" in output + assert "shared memory" not in output + + +def test_display_amd_dgpu_zero_vram_does_not_say_shared_memory(monkeypatch): + """An AMD dGPU with undetected VRAM should NOT be labelled + 'shared memory' — that would be a false positive.""" + from whichllm.output import _console as console_mod + from whichllm.output import display as display_mod + + buf = StringIO() + monkeypatch.setattr(console_mod, "console", Console(file=buf, force_terminal=False)) + + display_mod.display_hardware( + HardwareInfo( + gpus=[ + GPUInfo( + name="Navi 22 [Radeon RX 6750 XT]", + vendor="amd", + vram_bytes=0, + shared_memory=False, + ) + ], + cpu_name="CPU", + cpu_cores=8, + ram_bytes=32 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + ) + + output = buf.getvalue() + assert "shared memory" not in output diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..396f1e9 --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,47 @@ +"""Tests for resolving concrete downloadable artifacts.""" + +from whichllm.engine.types import CompatibilityResult +from whichllm.models.artifacts import attach_resolved_artifacts +from whichllm.models.types import GGUFVariant, ModelInfo + + +def test_attach_resolved_artifacts_maps_synthetic_quant_to_real_gguf_repo(): + base = ModelInfo( + id="Qwen/Qwen3-4B-Thinking-2507", + family_id="qwen3-4b-thinking", + name="Qwen3-4B-Thinking-2507", + parameter_count=4_000_000_000, + downloads=494_000, + ) + gguf_repo = ModelInfo( + id="MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF", + family_id="qwen3-4b-thinking", + name="Qwen3-4B-Thinking-2507-GGUF", + parameter_count=4_000_000_000, + downloads=26_000, + base_model="Qwen/Qwen3-4B-Thinking-2507", + gguf_variants=[ + GGUFVariant( + filename="Qwen3-4B-Thinking-2507-Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3-4B-Thinking-2507.Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2_000_000_000, + ) + result = CompatibilityResult( + model=base, + gguf_variant=synthetic, + can_run=True, + vram_required_bytes=3_000_000_000, + vram_available_bytes=8_000_000_000, + ) + + attach_resolved_artifacts([result], [base, gguf_repo]) + + assert result.artifact_model is gguf_repo + assert result.artifact_variant is gguf_repo.gguf_variants[0] diff --git a/tests/test_asahi_detection.py b/tests/test_asahi_detection.py new file mode 100644 index 0000000..6adeee2 --- /dev/null +++ b/tests/test_asahi_detection.py @@ -0,0 +1,213 @@ +"""Tests for Asahi Linux (Apple Silicon on Linux) detection — Issue #29.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from whichllm.hardware import apple, cpu + + +# ---- CPU name fallback for ARM Linux ---- + + +def test_cpu_name_lscpu_fallback(monkeypatch): + """When /proc/cpuinfo has no model name (ARM), lscpu should be tried.""" + # Simulate ARM /proc/cpuinfo (no model name field) + arm_cpuinfo = ( + "processor\t: 0\n" + "BogoMIPS\t: 48.00\n" + "Features\t: fp asimd evtstrm aes\n" + "CPU implementer\t: 0x61\n" + ) + monkeypatch.setattr("builtins.open", _fake_open(arm_cpuinfo)) + monkeypatch.setattr("platform.system", lambda: "Linux") + + lscpu_output = "Architecture: aarch64\nModel name: Apple M2\n" + + def fake_run(args, **kwargs): + if args == ["lscpu"]: + return subprocess.CompletedProcess(args, 0, stdout=lscpu_output, stderr="") + raise FileNotFoundError + + monkeypatch.setattr(cpu.subprocess, "run", fake_run) + + assert cpu.detect_cpu_name() == "Apple M2" + + +def test_cpu_name_devicetree_fallback(monkeypatch, tmp_path): + """When lscpu also fails, device tree model should be used.""" + arm_cpuinfo = "processor\t: 0\nFeatures\t: fp asimd\n" + monkeypatch.setattr("builtins.open", _fake_open(arm_cpuinfo)) + monkeypatch.setattr("platform.system", lambda: "Linux") + + # lscpu not available + def fake_run(args, **kwargs): + raise FileNotFoundError + + monkeypatch.setattr(cpu.subprocess, "run", fake_run) + + # Device tree has the machine model + dt_model = tmp_path / "model" + dt_model.write_bytes(b"Apple MacBook Air (M2, 2022)\x00") + monkeypatch.setattr( + cpu, "_cpu_name_from_devicetree", lambda: _read_dt_model(dt_model) + ) + + assert cpu.detect_cpu_name() == "Apple M2" + + +def test_cpu_name_devicetree_extracts_chip_variants(): + """Verify chip name extraction from various device tree model strings.""" + assert _extract("Apple MacBook Air (M2, 2022)") == "Apple M2" + assert _extract("Apple Mac Mini (M4 Pro, 2024)") == "Apple M4 Pro" + assert _extract("Apple Mac Studio (M2 Ultra, 2023)") == "Apple M2 Ultra" + assert _extract("Apple Mac Pro (M2 Max, 2023)") == "Apple M2 Max" + assert _extract("Apple MacBook Pro (M1, 2020)") == "Apple M1" + + +def test_cpu_name_devicetree_non_apple(): + """Non-Apple device tree models should not produce an Apple chip name.""" + assert _extract("Raspberry Pi 4 Model B Rev 1.5") is None + assert _extract("Qualcomm Snapdragon 8cx Gen 3") is None + + +def test_cpu_name_lscpu_ignores_dash(monkeypatch): + """lscpu returning '-' as the model name should be ignored.""" + arm_cpuinfo = "processor\t: 0\n" + monkeypatch.setattr("builtins.open", _fake_open(arm_cpuinfo)) + monkeypatch.setattr("platform.system", lambda: "Linux") + + lscpu_output = "Architecture: aarch64\nModel name: -\n" + + def fake_run(args, **kwargs): + if args == ["lscpu"]: + return subprocess.CompletedProcess(args, 0, stdout=lscpu_output, stderr="") + raise FileNotFoundError + + monkeypatch.setattr(cpu.subprocess, "run", fake_run) + monkeypatch.setattr(cpu, "_cpu_name_from_devicetree", lambda: "Apple M2") + + assert cpu.detect_cpu_name() == "Apple M2" + + +# ---- Apple GPU detection on Asahi Linux ---- + + +def test_detect_asahi_gpu_from_sysfs(monkeypatch, tmp_path): + """Asahi DRM driver should be detected and produce an Apple GPUInfo.""" + _setup_asahi_sysfs(tmp_path) + monkeypatch.setattr( + apple, + "_chip_name_from_devicetree", + lambda: "Apple M2", + ) + monkeypatch.setattr("psutil.virtual_memory", _fake_vmem(24 * 1024**3)) + + gpus = apple.detect_apple_gpu_linux(drm_path=tmp_path) + + assert len(gpus) == 1 + assert gpus[0].vendor == "apple" + assert gpus[0].name == "Apple M2" + assert gpus[0].vram_bytes == 24 * 1024**3 + assert gpus[0].shared_memory is True + assert gpus[0].memory_bandwidth_gbps == 100.0 # M2 bandwidth + + +def test_detect_asahi_gpu_fallback_name(monkeypatch, tmp_path): + """When device tree is unavailable, GPU should be named 'Apple Silicon'.""" + _setup_asahi_sysfs(tmp_path) + monkeypatch.setattr(apple, "_chip_name_from_devicetree", lambda: None) + monkeypatch.setattr("psutil.virtual_memory", _fake_vmem(8 * 1024**3)) + + gpus = apple.detect_apple_gpu_linux(drm_path=tmp_path) + + assert len(gpus) == 1 + assert gpus[0].name == "Apple Silicon" + + +def test_detect_asahi_gpu_ignores_non_apple_drivers(tmp_path): + """Non-Apple DRM drivers should not be detected as Apple GPUs.""" + card = tmp_path / "card0" / "device" / "driver" + card.mkdir(parents=True) + # Symlink to a non-Apple driver name + target = tmp_path / "drivers" / "amdgpu" + target.mkdir(parents=True) + (tmp_path / "card0" / "device" / "driver").rmdir() + (tmp_path / "card0" / "device" / "driver").symlink_to(target) + + assert apple.detect_apple_gpu_linux(drm_path=tmp_path) == [] + + +def test_detect_asahi_gpu_no_drm(tmp_path): + """When /sys/class/drm doesn't exist, return empty.""" + nonexistent = tmp_path / "no_drm" + assert apple.detect_apple_gpu_linux(drm_path=nonexistent) == [] + + +# ---- Helpers ---- + + +def _setup_asahi_sysfs(tmp_path: Path) -> None: + """Create a minimal sysfs tree mimicking the Asahi DRM driver.""" + device_dir = tmp_path / "card0" / "device" + device_dir.mkdir(parents=True) + # Symlink driver → .../asahi + driver_target = tmp_path / "drivers" / "asahi" + driver_target.mkdir(parents=True) + (device_dir / "driver").symlink_to(driver_target) + + +def _fake_open(content: str): + """Return a context manager that yields lines from *content* + when the path is /proc/cpuinfo, and delegates otherwise.""" + import builtins + import io + + real_open = builtins.open + + def patched_open(path, *a, **kw): + if str(path) == "/proc/cpuinfo": + return io.StringIO(content) + return real_open(path, *a, **kw) + + return patched_open + + +def _fake_vmem(total: int): + """Return a callable that mimics psutil.virtual_memory().""" + from collections import namedtuple + + Vmem = namedtuple("svmem", ["total"]) + + def _inner(): + return Vmem(total=total) + + return _inner + + +def _read_dt_model(path: Path) -> str | None: + """Simulate _cpu_name_from_devicetree reading from a custom path.""" + import re + + try: + raw = path.read_bytes() + model = raw.decode("utf-8", errors="replace").strip().rstrip("\x00") + if not model: + return None + m = re.search(r"\b(M\d+(?:\s+(?:Pro|Max|Ultra))?)\b", model) + if m: + return f"Apple {m.group(1)}" + return model + except OSError: + return None + + +def _extract(model: str) -> str | None: + """Run the chip-name regex against a model string.""" + import re + + m = re.search(r"\b(M\d+(?:\s+(?:Pro|Max|Ultra))?)\b", model) + if m: + return f"Apple {m.group(1)}" + return None diff --git a/tests/test_benchmark_lookup.py b/tests/test_benchmark_lookup.py new file mode 100644 index 0000000..c531b82 --- /dev/null +++ b/tests/test_benchmark_lookup.py @@ -0,0 +1,182 @@ +"""Tests for benchmark lookup direct/inherited semantics.""" + +import asyncio + +import whichllm.models.benchmark_sources as benchmark_sources +from whichllm.models.benchmark import ( + _lineage_recency_factor, + build_line_bucket_index, + build_score_index, + fetch_benchmark_scores, + lookup_benchmark, + lookup_benchmark_evidence, +) + + +def test_fetch_benchmark_scores_disables_brotli_accept_encoding(monkeypatch): + encodings: list[str] = [] + + async def fake_source(client): + encodings.append(client.headers["accept-encoding"]) + return {} + + monkeypatch.setattr( + benchmark_sources, "fetch_leaderboard_with_fallback", fake_source + ) + monkeypatch.setattr(benchmark_sources, "fetch_arena_scores", fake_source) + monkeypatch.setattr(benchmark_sources, "fetch_aa_index_scores", fake_source) + monkeypatch.setattr(benchmark_sources, "fetch_aider_polyglot_scores", fake_source) + monkeypatch.setattr(benchmark_sources, "fetch_vision_scores", fake_source) + monkeypatch.setattr(benchmark_sources, "get_livebench_data", lambda: {}) + + assert asyncio.run(fetch_benchmark_scores()) == {} + assert set(encodings) == {"gzip, deflate"} + + +def test_lookup_benchmark_model_id_match_is_direct(): + scores = {"Qwen/Qwen2.5-7B-Instruct": 70.0} + ci, line = build_score_index(scores) + result = lookup_benchmark( + "Qwen/Qwen2.5-7B-Instruct", + None, + scores, + ci, + line, + ) + assert result == (70.0, True) + + +def test_lookup_benchmark_base_model_match_is_inherited(): + scores = {"google/gemma-3-27b-it": 82.2} + ci, line = build_score_index(scores) + result = lookup_benchmark( + "ISTA-DASLab/gemma-3-27b-it-GPTQ-4b-128g", + "google/gemma-3-27b-it", + scores, + ci, + line, + ) + assert result == (82.2, False) + + +def test_lookup_benchmark_gguf_suffix_match_is_inherited(): + scores = {"Qwen/Qwen2.5-7B-Instruct": 70.0} + ci, line = build_score_index(scores) + result = lookup_benchmark( + "Qwen/Qwen2.5-7B-Instruct-GGUF", + None, + scores, + ci, + line, + ) + assert result == (70.0, False) + + +def test_lookup_benchmark_community_gguf_without_base_model_matches_official_id(): + scores = {"Qwen/Qwen3.6-27B": 83.5} + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "unsloth/Qwen3.6-27B-GGUF", + None, + scores, + ci, + line, + buckets, + ) + assert result.source == "variant" + assert result.score == 83.5 + + +def test_lookup_benchmark_community_gguf_underscore_name_matches_official_id(): + scores = {"Qwen/Qwen3.6-35B-A3B": 86.0} + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "unsloth/Qwen_Qwen3.6-35B-A3B-GGUF", + None, + scores, + ci, + line, + buckets, + ) + assert result.source == "variant" + assert result.score == 86.0 + + +def test_lookup_benchmark_community_gguf_keeps_params_guard(): + scores = {"Qwen/Qwen3.6-27B": 83.5} + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "unsloth/Qwen3.6-27B-GGUF", + None, + scores, + ci, + line, + buckets, + actual_params_b=6.6, + ) + assert result.source != "variant" + + +def test_lookup_benchmark_community_gguf_beats_self_reported_score(): + scores = {"Qwen/Qwen3.6-27B": 83.5} + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "unsloth/Qwen3.6-27B-GGUF", + None, + scores, + ci, + line, + buckets, + self_reported_score=12.0, + ) + assert result.source == "variant" + assert result.score == 83.5 + + +def test_lookup_benchmark_evidence_direct_has_full_confidence(): + scores = {"Qwen/Qwen2.5-7B-Instruct": 70.0} + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "Qwen/Qwen2.5-7B-Instruct", + None, + scores, + ci, + line, + buckets, + ) + assert result.source == "direct" + assert result.confidence == 1.0 + assert result.score == 70.0 + + +def test_lookup_benchmark_evidence_line_uses_size_aware_interpolation(): + scores = { + "Qwen/Qwen3-8B-Instruct": 65.0, + "Qwen/Qwen3-32B-Instruct": 85.0, + } + ci, line = build_score_index(scores) + buckets = build_line_bucket_index(scores) + result = lookup_benchmark_evidence( + "Qwen/Qwen3-14B-Instruct-GGUF", + None, + scores, + ci, + line, + buckets, + ) + assert result.source == "line_interp" + assert result.score is not None + assert 65.0 < result.score < 85.0 + assert result.confidence > 0.2 + + +def test_lineage_recency_t5gemma_variants_are_not_demoted_as_old_gemma(): + assert _lineage_recency_factor("google/t5gemma-4b") == 1.0 + assert _lineage_recency_factor("google/t5-gemma-4b") == 1.0 + assert _lineage_recency_factor("google/t5_gemma-4b") == 1.0 + assert _lineage_recency_factor("google/gemma-2-2b") < 1.0 diff --git a/tests/test_cache_encoding.py b/tests/test_cache_encoding.py new file mode 100644 index 0000000..53df9db --- /dev/null +++ b/tests/test_cache_encoding.py @@ -0,0 +1,65 @@ +"""Regression tests for cross-platform cache encoding.""" + +from __future__ import annotations + +import json +import time + +import whichllm.models.benchmark as benchmark_mod +import whichllm.models.cache as cache_mod + + +class _ReadableCacheFile: + def __init__(self, payload: dict): + self.payload = payload + self.encoding = None + + def exists(self) -> bool: + return True + + def read_text(self, *, encoding: str | None = None) -> str: + self.encoding = encoding + return json.dumps(self.payload, ensure_ascii=False) + + +class _WritableCacheFile: + def __init__(self): + self.encoding = None + self.text = None + + def write_text(self, text: str, *, encoding: str | None = None) -> int: + self.encoding = encoding + self.text = text + return len(text) + + +def test_model_cache_reads_and_writes_utf8(monkeypatch, tmp_path): + reader = _ReadableCacheFile( + {"cached_at": time.time(), "models": [{"id": "test/Omega-Ω"}]} + ) + monkeypatch.setattr(cache_mod, "CACHE_FILE", reader) + assert cache_mod.load_cache() == [{"id": "test/Omega-Ω"}] + assert reader.encoding == "utf-8" + + writer = _WritableCacheFile() + monkeypatch.setattr(cache_mod, "CACHE_DIR", tmp_path) + monkeypatch.setattr(cache_mod, "CACHE_FILE", writer) + cache_mod.save_cache([{"id": "test/Omega-Ω"}]) + assert writer.encoding == "utf-8" + assert "Ω" in writer.text + + +def test_benchmark_cache_reads_and_writes_utf8(monkeypatch, tmp_path): + reader = _ReadableCacheFile( + {"cached_at": time.time(), "scores": {"test/Omega-Ω": 1.0}} + ) + monkeypatch.setattr(benchmark_mod, "BENCHMARK_CACHE", reader) + assert benchmark_mod.load_benchmark_cache() == {"test/Omega-Ω": 1.0} + assert reader.encoding == "utf-8" + + writer = _WritableCacheFile() + monkeypatch.setattr(benchmark_mod, "CACHE_DIR", tmp_path) + monkeypatch.setattr(benchmark_mod, "BENCHMARK_CACHE", writer) + benchmark_mod.save_benchmark_cache({"test/Omega-Ω": 1.0}) + assert writer.encoding == "utf-8" + assert "Ω" in writer.text diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..5ebf8fb --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,1381 @@ +"""Tests for CLI helper logic.""" + +import httpx +import pytest +from typer import Exit + +import whichllm.cli as cli_mod +import whichllm.__main__ as main_mod +from whichllm.cli import ( + _apply_memory_budgets, + _apply_gpu_overrides, + _auto_min_params_for_profile, + _extract_id_size_b, + _fill_missing_published_at, + _format_fetch_error, + _generate_chat_script, + _include_vision_candidates, + _merge_model_eval_benchmarks, + _parse_memory_amount, + _pick_gguf_variant, + _resolve_ranked_gguf_for_run, + _resolve_evidence_mode, + _resolve_fit_filter, + _resolve_speed_filter, + _search_model, + _validate_evidence, + _validate_gpu_flags, + app, +) +from whichllm.utils import _current_version +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo +from typer.testing import CliRunner + + +def _hw_with_gpu(vram_gb: int) -> HardwareInfo: + return HardwareInfo( + gpus=[ + GPUInfo( + name="GPU", + vendor="nvidia", + vram_bytes=vram_gb * 1024**3, + memory_bandwidth_gbps=1.0, + ) + ], + cpu_name="CPU", + cpu_cores=1, + ram_bytes=16 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + + +def test_auto_min_params_general_by_vram(): + # Updated thresholds: tiny GPUs (4-8GB) get a lower floor so they can + # surface full-GPU 3-4B models instead of being forced into 7B+ + # partial-offload-only candidates. + assert _auto_min_params_for_profile(_hw_with_gpu(4), "general") == 2.0 + assert _auto_min_params_for_profile(_hw_with_gpu(6), "general") == 3.0 + assert _auto_min_params_for_profile(_hw_with_gpu(8), "general") == 5.0 + assert _auto_min_params_for_profile(_hw_with_gpu(12), "general") == 8.0 + assert _auto_min_params_for_profile(_hw_with_gpu(24), "general") == 10.0 + assert _auto_min_params_for_profile(_hw_with_gpu(32), "general") == 12.0 + + +def test_auto_min_params_non_general_disabled(): + assert _auto_min_params_for_profile(_hw_with_gpu(24), "coding") is None + + +def test_auto_min_params_uses_usable_vram_budget(): + hw = _hw_with_gpu(20) + hw.gpus[0].usable_vram_bytes = int(19.0 * 1024**3) + + assert _auto_min_params_for_profile(hw, "general") == 8.0 + + +def test_auto_min_params_uses_ram_budget_for_shared_memory_gpu(): + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Apple M2", + vendor="apple", + vram_bytes=16 * 1024**3, + usable_vram_bytes=15 * 1024**3, + shared_memory=True, + ) + ], + ram_bytes=16 * 1024**3, + ram_budget_bytes=4 * 1024**3, + ) + + assert _auto_min_params_for_profile(hw, "general") == 2.0 + + +def test_apply_gpu_overrides_accepts_multiple_simulated_gpus(): + hw = HardwareInfo(gpus=[], ram_bytes=64 * 1024**3, os="linux") + + _apply_gpu_overrides(hw, cpu_only=False, gpu=["2x RTX 4090"], vram=None) + + assert len(hw.gpus) == 2 + assert all(gpu.vendor == "nvidia" for gpu in hw.gpus) + assert all(gpu.vram_bytes == 24 * 1024**3 for gpu in hw.gpus) + + +def test_validate_gpu_flags_allows_detected_vram_override(): + _validate_gpu_flags(cpu_only=False, gpu=None, vram=8.0, bandwidth=None) + + +def test_validate_gpu_flags_rejects_non_positive_overrides(): + with pytest.raises(Exit): + _validate_gpu_flags(cpu_only=False, gpu=None, vram=0, bandwidth=None) + with pytest.raises(Exit): + _validate_gpu_flags(cpu_only=False, gpu=None, vram=None, bandwidth=-1) + + +def test_validate_gpu_flags_rejects_gpu_index_with_simulated_gpu(): + with pytest.raises(Exit): + _validate_gpu_flags( + cpu_only=False, + gpu=["RTX 4090"], + vram=24.0, + bandwidth=None, + gpu_index=0, + ) + + +def test_apply_gpu_overrides_updates_detected_shared_memory_gpu(): + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Intel UHD Graphics", + vendor="intel", + vram_bytes=0, + shared_memory=True, + memory_bandwidth_gbps=None, + ) + ], + cpu_name="CPU", + cpu_cores=8, + ram_bytes=32 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + + _apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=6.0, bandwidth=88.5) + + assert hw.gpus[0].vram_bytes == 6 * 1024**3 + assert hw.gpus[0].usable_vram_bytes is None + assert hw.gpus[0].memory_bandwidth_gbps == 88.5 + assert hw.gpus[0].shared_memory is True + assert hw.gpus[0].vram_overridden is True + + +def test_apply_gpu_overrides_updates_selected_detected_gpu_only(): + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA RTX 4060", + vendor="nvidia", + vram_bytes=8 * 1024**3, + memory_bandwidth_gbps=272.0, + ), + GPUInfo( + name="Intel UHD Graphics", + vendor="intel", + vram_bytes=0, + shared_memory=True, + ), + ], + cpu_name="CPU", + cpu_cores=8, + ram_bytes=32 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + + _apply_gpu_overrides( + hw, cpu_only=False, gpu=None, vram=4.0, bandwidth=60.0, gpu_index=1 + ) + + assert hw.gpus[0].vram_bytes == 8 * 1024**3 + assert hw.gpus[0].memory_bandwidth_gbps == 272.0 + assert hw.gpus[1].vram_bytes == 4 * 1024**3 + assert hw.gpus[1].memory_bandwidth_gbps == 60.0 + assert hw.gpus[1].vram_overridden is True + + +def test_apply_gpu_overrides_requires_gpu_index_for_multiple_detected_gpus(): + hw = HardwareInfo( + gpus=[ + GPUInfo(name="GPU 0", vendor="nvidia", vram_bytes=8 * 1024**3), + GPUInfo(name="GPU 1", vendor="intel", vram_bytes=0, shared_memory=True), + ], + cpu_name="CPU", + cpu_cores=8, + ram_bytes=32 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + + with pytest.raises(Exit): + _apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=4.0) + + +def test_apply_gpu_overrides_updates_simulated_gpu_bandwidth(): + hw = HardwareInfo(gpus=[], ram_bytes=32 * 1024**3, os="linux") + + _apply_gpu_overrides( + hw, cpu_only=False, gpu=["Unknown GPU"], vram=4.0, bandwidth=72.0 + ) + + assert len(hw.gpus) == 1 + assert hw.gpus[0].vram_bytes == 4 * 1024**3 + assert hw.gpus[0].memory_bandwidth_gbps == 72.0 + + +def test_apply_gpu_overrides_rejects_override_without_gpu(): + hw = HardwareInfo(gpus=[], ram_bytes=32 * 1024**3, os="linux") + + with pytest.raises(Exit): + _apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=4.0) + + +def test_include_vision_candidates_by_profile(): + assert _include_vision_candidates("vision") is True + assert _include_vision_candidates("any") is True + assert _include_vision_candidates("general") is False + assert _include_vision_candidates("coding") is False + + +def test_fill_missing_published_at_updates_models(): + model = ModelInfo( + id="Qwen/Qwen3-8B-AWQ", + family_id="qwen3-8b", + name="Qwen3-8B-AWQ", + parameter_count=8_000_000_000, + downloads=1, + likes=1, + ) + result = CompatibilityResult( + model=model, + gguf_variant=None, + can_run=True, + vram_required_bytes=0, + vram_available_bytes=0, + ) + + async def _fake_fetch(ids: list[str]) -> dict[str, str]: + assert ids == ["Qwen/Qwen3-8B-AWQ"] + return {"Qwen/Qwen3-8B-AWQ": "2026-03-05T08:00:00.000Z"} + + updated = _fill_missing_published_at([model], [result], _fake_fetch) + assert updated is True + assert model.published_at == "2026-03-05T08:00:00.000Z" + + +def test_version_option_prints_version_and_exits(): + runner = CliRunner() + result = runner.invoke(app, ["--version"]) + assert result.exit_code == 0 + assert _current_version() in result.stdout + + +def test_module_entrypoint_uses_cli_app(): + assert main_mod.app is app + + +def test_format_fetch_error_uses_exception_class_when_message_is_empty(): + class EmptyNetworkError(Exception): + def __str__(self) -> str: + return "" + + assert _format_fetch_error(EmptyNetworkError()) == ( + "EmptyNetworkError with no detail from the network layer" + ) + + +def test_format_fetch_error_includes_status_and_url_for_empty_http_error(): + request = httpx.Request("GET", "https://huggingface.co/api/models") + response = httpx.Response(429, request=request) + error = httpx.HTTPStatusError("", request=request, response=response) + + assert _format_fetch_error(error) == ( + "HTTPStatusError: HTTP 429 for https://huggingface.co/api/models" + ) + + +def test_merge_model_eval_benchmarks_is_now_a_noop(): + """As of the self_reported evidence tier, _merge_model_eval_benchmarks + must NOT mutate the leaderboard scores. Uploader-reported hf_eval values + are consumed directly by the ranker as a separate, low-trust source. + """ + model_direct_missing = ModelInfo( + id="meta-llama/Llama-3.1-8B-Instruct", + family_id="llama-3.1-8b", + name="Llama-3.1-8B-Instruct", + parameter_count=8_000_000_000, + downloads=1, + likes=1, + benchmark_scores={"hf_eval": 66.4}, + ) + model_already_present = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1, + likes=1, + benchmark_scores={"hf_eval": 70.0}, + ) + original = {"Qwen/Qwen2.5-7B-Instruct": 71.2} + merged, injected = _merge_model_eval_benchmarks( + [model_direct_missing, model_already_present], + original, + ) + # Function is a deprecation no-op now. + assert injected == 0 + assert merged is original or merged == original + # Critically, the uploader-reported value MUST NOT have been injected + # under the model id, because doing so would make it appear as a + # direct leaderboard hit. + assert "meta-llama/Llama-3.1-8B-Instruct" not in merged + + +def test_validate_evidence_accepts_all_modes(): + assert _validate_evidence("strict") == "strict" + assert _validate_evidence("base") == "base" + assert _validate_evidence("any") == "any" + + +def test_validate_evidence_rejects_unknown_mode(): + with pytest.raises(Exit): + _validate_evidence("foo") + + +def test_resolve_evidence_mode_direct_alias_wins(): + assert _resolve_evidence_mode("base", direct=True) == "strict" + + +def test_resolve_fit_filter_accepts_gpu_only_alias(): + assert _resolve_fit_filter("any", gpu_only=False) == "any" + assert _resolve_fit_filter("gpu", gpu_only=False) == "full_gpu" + assert _resolve_fit_filter("full-gpu", gpu_only=False) == "full_gpu" + assert _resolve_fit_filter("full_gpu", gpu_only=False) == "full_gpu" + assert _resolve_fit_filter("any", gpu_only=True) == "full_gpu" + + +def test_resolve_fit_filter_rejects_unknown_mode(): + with pytest.raises(Exit): + _resolve_fit_filter("partial", gpu_only=False) + + +def test_resolve_speed_filter_presets_and_min_speed_override(): + assert _resolve_speed_filter("any", min_speed=None) is None + assert _resolve_speed_filter("usable", min_speed=None) == 10.0 + assert _resolve_speed_filter("fast", min_speed=None) == 30.0 + assert _resolve_speed_filter("fast", min_speed=2.5) == 2.5 + + +def test_resolve_speed_filter_rejects_unknown_mode(): + with pytest.raises(Exit): + _resolve_speed_filter("slowish", min_speed=None) + + +def test_parse_memory_amount_supports_gb_mb_and_percent(): + assert _parse_memory_amount("1.5GB", option_name="--x") == int(1.5 * 1024**3) + assert _parse_memory_amount("512MB", option_name="--x") == 512 * 1024**2 + assert _parse_memory_amount("8", option_name="--x") == 8 * 1024**3 + assert ( + _parse_memory_amount("10%", option_name="--x", total_bytes=20 * 1024**3) + == 2 * 1024**3 + ) + + +def test_apply_memory_budgets_sets_vram_headroom_and_ram_budget(): + hw = _hw_with_gpu(16) + + _apply_memory_budgets(hw, vram_headroom="1GB", ram_budget="8GB") + + assert hw.gpus[0].vram_bytes == 16 * 1024**3 + assert hw.gpus[0].usable_vram_bytes == 15 * 1024**3 + assert hw.ram_budget_bytes == 8 * 1024**3 + assert any("VRAM headroom" in note for note in hw.budget_notes) + assert any("RAM budget" in note for note in hw.budget_notes) + + +def test_apply_memory_budgets_validates_vram_headroom_without_gpus(): + hw = HardwareInfo(gpus=[], ram_bytes=16 * 1024**3) + + with pytest.raises(Exit): + _apply_memory_budgets(hw, vram_headroom="nope", ram_budget=None) + + +def test_apply_memory_budgets_accepts_valid_noop_vram_headroom_without_gpus(): + hw = HardwareInfo(gpus=[], ram_bytes=16 * 1024**3) + + _apply_memory_budgets(hw, vram_headroom="10%", ram_budget=None) + + assert hw.gpus == [] + assert hw.ram_budget_bytes is None + + +def test_main_passes_gpu_only_fit_filter(monkeypatch): + model = ModelInfo( + id="org/Test-7B", + family_id="test-7b", + name="Test-7B", + parameter_count=7_000_000_000, + downloads=1, + likes=1, + published_at="2026-01-01T00:00:00.000Z", + ) + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + captured["fit_filter"] = kwargs.get("fit_filter") + return [ + CompatibilityResult( + model=model, + gguf_variant=None, + can_run=True, + vram_required_bytes=4 * 1024**3, + vram_available_bytes=8 * 1024**3, + fit_type="full_gpu", + quality_score=80.0, + ) + ] + + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr( + "whichllm.output.display.display_hardware", lambda hardware: None + ) + monkeypatch.setattr( + "whichllm.output.display.display_ranking", + lambda results, **kwargs: None, + ) + + result = CliRunner().invoke(app, ["--gpu-only"]) + + assert result.exit_code == 0 + assert captured["fit_filter"] == "full_gpu" + + +def test_main_passes_speed_preset_and_default_runtime_columns(monkeypatch): + model = ModelInfo( + id="org/Test-7B", + family_id="test-7b", + name="Test-7B", + parameter_count=7_000_000_000, + downloads=1, + likes=1, + published_at="2026-01-01T00:00:00.000Z", + ) + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + captured["min_speed"] = kwargs.get("min_speed") + return [ + CompatibilityResult( + model=model, + gguf_variant=None, + can_run=True, + vram_required_bytes=4 * 1024**3, + vram_available_bytes=8 * 1024**3, + fit_type="full_gpu", + estimated_tok_per_sec=8.0, + quality_score=80.0, + ) + ] + + def fake_display_ranking(results, **kwargs): + captured["show_status"] = kwargs.get("show_status") + + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr( + "whichllm.output.display.display_hardware", lambda hardware: None + ) + monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking) + + result = CliRunner().invoke(app, ["--speed", "usable"]) + + assert result.exit_code == 0 + assert captured["min_speed"] == 10.0 + assert captured["show_status"] is True + + +def test_main_details_flag_restores_metadata_columns(monkeypatch): + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + return [] + + def fake_display_ranking(results, **kwargs): + captured["show_status"] = kwargs.get("show_status") + + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr( + "whichllm.output.display.display_hardware", lambda hardware: None + ) + monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking) + + result = CliRunner().invoke(app, ["--details", "--min-params", "1"]) + + assert result.exit_code == 0 + assert captured["show_status"] is False + + +def test_main_markdown_alias_dispatches_markdown_output(monkeypatch): + model = ModelInfo( + id="org/Test-7B", + family_id="test-7b", + name="Test-7B", + parameter_count=7_000_000_000, + downloads=1, + likes=1, + published_at="2026-01-01T00:00:00.000Z", + ) + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + return [ + CompatibilityResult( + model=model, + gguf_variant=None, + can_run=True, + vram_required_bytes=4 * 1024**3, + vram_available_bytes=8 * 1024**3, + fit_type="full_gpu", + estimated_tok_per_sec=12.0, + quality_score=80.0, + ) + ] + + def fake_display_markdown(results, hardware, **kwargs): + captured["called"] = True + captured["show_status"] = kwargs.get("show_status") + + def fail_display_hardware(hardware): + raise AssertionError("markdown output should not render Rich hardware panel") + + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr( + "whichllm.output.display.display_markdown", fake_display_markdown + ) + monkeypatch.setattr( + "whichllm.output.display.display_hardware", fail_display_hardware + ) + + result = CliRunner().invoke(app, ["-m"]) + + assert result.exit_code == 0 + assert captured["called"] is True + assert captured["show_status"] is True + + +def test_main_json_and_markdown_are_mutually_exclusive(): + result = CliRunner().invoke(app, ["--json", "--markdown"]) + + assert result.exit_code == 1 + assert "--json and --markdown are mutually exclusive" in result.stdout + + +def test_main_top_zero_rejected(): + result = CliRunner().invoke(app, ["--top", "0"]) + + assert result.exit_code == 1 + assert "--top must be 1 or greater" in result.stdout + + +def test_main_top_negative_rejected(): + # ``results[:-5]`` would silently drop the best-ranked models. + result = CliRunner().invoke(app, ["--top=-5"]) + + assert result.exit_code == 1 + assert "--top must be 1 or greater" in result.stdout + + +def test_main_negative_min_speed_rejected(): + result = CliRunner().invoke(app, ["--min-speed=-1"]) + + assert result.exit_code == 1 + assert "--min-speed must be 0 or greater" in result.stdout + + +def test_main_negative_min_params_rejected(): + result = CliRunner().invoke(app, ["--min-params=-2"]) + + assert result.exit_code == 1 + assert "--min-params must be 0 or greater" in result.stdout + + +def test_validate_ranking_flags_accepts_valid_values(): + # A valid combination must not raise (no recommendations are dropped). + cli_mod._validate_ranking_flags(top=10, min_speed=0.0, min_params=None) + cli_mod._validate_ranking_flags(top=1, min_speed=None, min_params=7.0) + + +def test_upgrade_top_zero_rejected(): + # The upgrade command shares the same --top -> results[:top_n] hazard. + result = CliRunner().invoke(app, ["upgrade", "RTX 4090", "--top", "0"]) + + assert result.exit_code == 1 + assert "--top must be 1 or greater" in result.stdout + + +def test_main_empty_gpu_only_result_shows_fit_message(monkeypatch): + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + return [] + + def fake_display_ranking(results, **kwargs): + captured["empty_message"] = kwargs.get("empty_message") + + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr( + "whichllm.output.display.display_hardware", lambda hardware: None + ) + monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking) + + result = CliRunner().invoke(app, ["--fit", "full-gpu", "--min-params", "1"]) + + assert result.exit_code == 0 + assert "No full-GPU models found" in captured["empty_message"] + + +# --------------- plan command tests --------------- + + +def test_plan_no_model_found_shows_error(monkeypatch): + monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: []) + runner = CliRunner() + result = runner.invoke(app, ["plan", "nonexistent_model_xyz_999"]) + assert result.exit_code != 0 + assert "No model found" in result.stdout + + +def test_plan_display_plan_renders_tables(): + """display_plan should render model info, VRAM table, and GPU table.""" + from whichllm.output.display import display_plan + + model = ModelInfo( + id="test-org/Test-Model-7B-GGUF", + family_id="test-7b", + name="Test-Model-7B", + parameter_count=7_000_000_000, + architecture="llama", + context_length=4096, + license="mit", + downloads=100, + likes=10, + ) + # Should not raise + display_plan(model, context_length=4096, target_quant="Q4_K_M") + + +def test_plan_display_plan_json_outputs_valid_json(): + """display_plan_json should output valid JSON.""" + import json as json_mod + from io import StringIO + + from rich.console import Console + + from whichllm.output.display import display_plan_json + + model = ModelInfo( + id="test-org/Test-Model-7B-GGUF", + family_id="test-7b", + name="Test-Model-7B", + parameter_count=7_000_000_000, + architecture="llama", + context_length=4096, + license="mit", + downloads=100, + likes=10, + ) + # Capture output + buf = StringIO() + import whichllm.output._console as console_mod + + orig_console = console_mod.console + console_mod.console = Console(file=buf, force_terminal=False) + try: + display_plan_json(model, context_length=4096, target_quant="Q4_K_M") + finally: + console_mod.console = orig_console + raw = buf.getvalue().strip() + data = json_mod.loads(raw) + assert data["model"]["id"] == "test-org/Test-Model-7B-GGUF" + assert "vram_by_quant" in data + assert "gpu_compatibility" in data + assert data["target_quant"] == "Q4_K_M" + + +# --------------- helper tests --------------- + + +def _make_model( + model_id="org/Test-7B-GGUF", + downloads=100, + gguf_variants=None, + parameter_count=7_000_000_000, +): + return ModelInfo( + id=model_id, + family_id="test-7b", + name="Test-7B", + parameter_count=parameter_count, + downloads=downloads, + likes=10, + gguf_variants=gguf_variants or [], + ) + + +def test_search_model_exact_match(): + models = [_make_model("org/Llama-8B"), _make_model("org/Qwen-7B")] + result = _search_model(models, "org/Llama-8B") + assert result.id == "org/Llama-8B" + + +def test_search_model_endswith_match(): + models = [_make_model("org/Llama-8B"), _make_model("org/Qwen-7B")] + result = _search_model(models, "Llama-8B") + assert result.id == "org/Llama-8B" + + +def test_search_model_term_match(): + models = [_make_model("org/Llama-3.1-8B-GGUF"), _make_model("org/Qwen-7B")] + result = _search_model(models, "llama 8b") + assert result.id == "org/Llama-3.1-8B-GGUF" + + +def test_search_model_not_found(): + models = [_make_model("org/Llama-8B")] + with pytest.raises(Exit): + _search_model(models, "nonexistent_xyz") + + +# --- regression tests for size-token substring matching (#107) --- + + +def test_search_model_7b_does_not_match_1_7b(): + """'qwen 7b' should NOT match Qwen3-1.7B (issue #107).""" + models = [ + _make_model( + "org/Qwen3-1.7B-GGUF", downloads=9999, parameter_count=1_700_000_000 + ), + _make_model("org/Qwen3-7B-GGUF", downloads=100, parameter_count=7_000_000_000), + ] + result = _search_model(models, "qwen 7b") + assert result.id == "org/Qwen3-7B-GGUF" + + +def test_search_model_2b_does_not_match_12b(): + """'gemma 2b' should NOT match gemma-3-12b-it (issue #107).""" + models = [ + _make_model( + "google/gemma-3-12b-it", downloads=5000, parameter_count=12_000_000_000 + ), + _make_model("google/gemma-2b", downloads=100, parameter_count=2_000_000_000), + ] + result = _search_model(models, "gemma 2b") + assert result.id == "google/gemma-2b" + + +def test_search_model_3b_does_not_match_30b_a3b(): + """'qwen 3b' should NOT match Qwen3-30B-A3B (issue #107).""" + models = [ + _make_model( + "org/Qwen3-30B-A3B-GGUF", downloads=8000, parameter_count=30_000_000_000 + ), + _make_model("org/Qwen3-3B-GGUF", downloads=50, parameter_count=3_000_000_000), + ] + result = _search_model(models, "qwen 3b") + assert result.id == "org/Qwen3-3B-GGUF" + + +def test_search_model_no_size_token_still_works(): + """Queries without a size token should still use plain substring matching.""" + models = [ + _make_model("org/Llama-3.1-8B-GGUF", parameter_count=8_000_000_000), + _make_model("org/Qwen-7B", parameter_count=7_000_000_000), + ] + result = _search_model(models, "llama gguf") + assert result.id == "org/Llama-3.1-8B-GGUF" + + +def test_search_model_millions_size_token(): + """'500m' size token should match a ~500M parameter model.""" + models = [ + _make_model("org/SmolLM-500M", downloads=200, parameter_count=500_000_000), + _make_model("org/SmolLM-1.7B", downloads=300, parameter_count=1_700_000_000), + ] + result = _search_model(models, "smollm 500m") + assert result.id == "org/SmolLM-500M" + + +def test_search_model_decimal_size_token(): + """'0.5b' size token should match a ~500M parameter model.""" + models = [ + _make_model("org/TinyModel-0.5B", downloads=100, parameter_count=500_000_000), + _make_model("org/TinyModel-3B", downloads=500, parameter_count=3_000_000_000), + ] + result = _search_model(models, "tinymodel 0.5b") + assert result.id == "org/TinyModel-0.5B" + + +def test_search_model_zero_param_count_passes_through(): + """Models with parameter_count=0 (missing metadata) should not be excluded.""" + models = [ + _make_model("org/Mystery-7B-GGUF", downloads=500, parameter_count=0), + ] + result = _search_model(models, "mystery 7b") + assert result.id == "org/Mystery-7B-GGUF" + + +def test_search_model_first_size_token_wins(): + """When multiple size tokens appear, only the first is used as a filter.""" + models = [ + _make_model( + "org/Qwen3-30B-A3B-GGUF", downloads=8000, parameter_count=30_000_000_000 + ), + _make_model( + "org/Qwen3-7B-3B-GGUF", downloads=100, parameter_count=7_000_000_000 + ), + ] + # "7b" is the first size token and filters by ~7B; "3b" becomes a text term + result = _search_model(models, "qwen 7b 3b") + assert result.id == "org/Qwen3-7B-3B-GGUF" + + +def test_search_model_7b_prefers_closest_size_over_downloads(): + """'qwen 7b' should pick 8B (closest to 7B) over 4B even if 4B has more downloads.""" + models = [ + _make_model("org/Qwen3-4B-GGUF", downloads=9000, parameter_count=4_000_000_000), + _make_model("org/Qwen3-8B-GGUF", downloads=100, parameter_count=8_000_000_000), + ] + result = _search_model(models, "qwen 7b") + assert result.id == "org/Qwen3-8B-GGUF" + + +def test_search_model_3b_prefers_closest_size_over_downloads(): + """'qwen 3b' should pick 3B (exact) over 4B even if 4B has more downloads.""" + models = [ + _make_model("org/Qwen3-4B-GGUF", downloads=9000, parameter_count=4_000_000_000), + _make_model("org/Qwen3-3B-GGUF", downloads=50, parameter_count=3_000_000_000), + ] + result = _search_model(models, "qwen 3b") + assert result.id == "org/Qwen3-3B-GGUF" + + +def test_search_model_size_tiebreak_falls_back_to_downloads(): + """When two models are equally close in size, prefer the one with more downloads.""" + models = [ + _make_model( + "org/Qwen3-8B-A-GGUF", downloads=100, parameter_count=8_000_000_000 + ), + _make_model( + "org/Qwen3-8B-B-GGUF", downloads=500, parameter_count=8_000_000_000 + ), + ] + result = _search_model(models, "qwen 7b") + assert result.id == "org/Qwen3-8B-B-GGUF" + + +def test_search_model_unknown_param_count_ranks_after_known(): + """Models with parameter_count=0 should rank after models with known sizes.""" + models = [ + _make_model("org/Qwen3-Unknown-GGUF", downloads=9999, parameter_count=0), + _make_model("org/Qwen3-7B-GGUF", downloads=10, parameter_count=7_000_000_000), + ] + result = _search_model(models, "qwen 7b") + assert result.id == "org/Qwen3-7B-GGUF" + + +@pytest.mark.parametrize( + "model_id, expected", + [ + ("org/Qwen3-8B-GGUF", 8.0), + ("org/Qwen3.5-9B-NVFP4", 9.0), + ("org/Qwen3-30B-A3B-GGUF", 30.0), + ("org/SmolLM-500M", 0.5), + ("org/TinyModel-1.7B-Chat", 1.7), + ("org/Llama-3.1-8B-Instruct", 8.0), + ("org/NoSizeLabel-GGUF", None), + ], +) +def test_extract_id_size_b(model_id, expected): + assert _extract_id_size_b(model_id) == expected + + +def test_search_model_7b_prefers_id_label_over_param_count(): + """'qwen 7b' should not pick a 9B-labeled repo even if its param count is closer.""" + models = [ + _make_model( + "org/Qwen3.5-9B-NVFP4", downloads=500, parameter_count=6_600_000_000 + ), + _make_model("org/Qwen3-8B-GGUF", downloads=100, parameter_count=8_000_000_000), + ] + result = _search_model(models, "qwen 7b") + assert result.id == "org/Qwen3-8B-GGUF" + + +def test_pick_gguf_variant_by_preference(): + variants = [ + GGUFVariant(filename="q2.gguf", quant_type="Q2_K", file_size_bytes=1000), + GGUFVariant(filename="q4km.gguf", quant_type="Q4_K_M", file_size_bytes=2000), + ] + model = _make_model(gguf_variants=variants) + result = _pick_gguf_variant(model) + assert result.quant_type == "Q4_K_M" + + +def test_pick_gguf_variant_with_filter(): + variants = [ + GGUFVariant(filename="q2.gguf", quant_type="Q2_K", file_size_bytes=1000), + GGUFVariant(filename="q4km.gguf", quant_type="Q4_K_M", file_size_bytes=2000), + ] + model = _make_model(gguf_variants=variants) + result = _pick_gguf_variant(model, quant_filter="Q2_K") + assert result.quant_type == "Q2_K" + + +def test_pick_gguf_variant_no_variants(): + model = _make_model(gguf_variants=[]) + result = _pick_gguf_variant(model) + assert result is None + + +def test_resolve_ranked_synthetic_gguf_to_real_repo(): + selected = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + downloads=50_000, + ) + real_gguf = ModelInfo( + id="unsloth/Qwen3.6-27B-GGUF", + family_id="qwen3-27b", + name="Qwen3.6-27B-GGUF", + parameter_count=27_000_000_000, + downloads=200_000, + base_model="Qwen/Qwen3.6-27B", + gguf_variants=[ + GGUFVariant( + filename="Qwen3.6-27B-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3.6-27B.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + + resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, real_gguf]) + + assert resolved is not None + model, variant = resolved + assert model.id == "unsloth/Qwen3.6-27B-GGUF" + assert variant.filename == "Qwen3.6-27B-Q4_K_M.gguf" + + +def test_resolve_ranked_synthetic_gguf_prefers_exact_quant(): + selected = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + ) + q5_only = ModelInfo( + id="converter/Qwen3.6-27B-GGUF", + family_id="qwen3-27b", + name="Qwen3.6-27B-GGUF", + parameter_count=27_000_000_000, + downloads=1_000_000, + gguf_variants=[ + GGUFVariant( + filename="q5.gguf", + quant_type="Q5_K_M", + file_size_bytes=18_000_000_000, + ) + ], + ) + q4_match = ModelInfo( + id="smaller/Qwen3.6-27B-GGUF", + family_id="qwen3-27b", + name="Qwen3.6-27B-GGUF", + parameter_count=27_000_000_000, + downloads=10, + gguf_variants=[ + GGUFVariant( + filename="q4.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3.6-27B.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + + resolved = _resolve_ranked_gguf_for_run( + selected, + synthetic, + [selected, q5_only, q4_match], + ) + + assert resolved is not None + model, variant = resolved + assert model.id == "smaller/Qwen3.6-27B-GGUF" + assert variant.quant_type == "Q4_K_M" + + +def test_resolve_ranked_synthetic_gguf_rejects_quant_mismatch(): + selected = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + ) + q5_only = ModelInfo( + id="converter/Qwen3.6-27B-GGUF", + family_id="qwen3-27b", + name="Qwen3.6-27B-GGUF", + parameter_count=27_000_000_000, + downloads=1_000_000, + gguf_variants=[ + GGUFVariant( + filename="q5.gguf", + quant_type="Q5_K_M", + file_size_bytes=18_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3.6-27B.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + + resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, q5_only]) + + assert resolved is None + + +def test_resolve_ranked_synthetic_gguf_without_real_repo_returns_none(): + selected = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + ) + unrelated = ModelInfo( + id="other/Model-7B-GGUF", + family_id="model-7b", + name="Model-7B-GGUF", + parameter_count=7_000_000_000, + gguf_variants=[ + GGUFVariant( + filename="other.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3.6-27B.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + + assert ( + _resolve_ranked_gguf_for_run(selected, synthetic, [selected, unrelated]) is None + ) + + +def test_resolve_ranked_synthetic_gguf_rejects_size_mismatch(): + selected = ModelInfo( + id="deepseek-ai/DeepSeek-V4-Flash", + family_id="deepseek-v4-flash", + name="DeepSeek-V4-Flash", + parameter_count=158_000_000_000, + ) + mtp_head = ModelInfo( + id="converter/deepseek-v4-flash-mtp-gguf", + family_id="deepseek-v4-flash", + name="DeepSeek-V4-Flash-MTP-GGUF", + parameter_count=6_600_000_000, + gguf_variants=[ + GGUFVariant( + filename="mtp.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="DeepSeek-V4-Flash.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=90_000_000_000, + ) + + resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, mtp_head]) + + assert resolved is None + + +# --------------- run/snippet command tests --------------- + + +def test_run_exits_gracefully(): + """run should fail gracefully (uv missing, or no model found).""" + runner = CliRunner() + result = runner.invoke(app, ["run", "some-model"]) + if result.exit_code != 0: + assert any( + msg in result.stdout + for msg in ("uv is required", "No model found", "llama-cpp-python") + ) + + +def test_transformers_chat_script_passes_tokenizer_mapping_to_generate(): + model = _make_model(model_id="org/Test-7B") + + script = _generate_chat_script( + model, variant=None, context_length=4096, cpu_only=False + ) + + assert "return_dict=True" in script + assert "kwargs=dict(**inputs, max_new_tokens=512, streamer=streamer)" in script + assert "kwargs=dict(input_ids=inputs" not in script + + +def test_transformers_chat_script_provides_disk_offload_folder(): + model = _make_model(model_id="org/Test-7B") + + script = _generate_chat_script( + model, variant=None, context_length=4096, cpu_only=False + ) + + assert 'tempfile.mkdtemp(prefix="whichllm_transformers_offload_")' in script + assert "offload_folder=offload_folder" in script + assert "shutil.rmtree(offload_folder, ignore_errors=True)" in script + + +def test_run_auto_pick_resolves_ranked_gguf_before_launch(monkeypatch): + selected = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + downloads=50_000, + ) + real_gguf = ModelInfo( + id="unsloth/Qwen3.6-27B-GGUF", + family_id="qwen3-27b", + name="Qwen3.6-27B-GGUF", + parameter_count=27_000_000_000, + downloads=200_000, + base_model="Qwen/Qwen3.6-27B", + gguf_variants=[ + GGUFVariant( + filename="q4.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + ], + ) + synthetic = GGUFVariant( + filename="Qwen3.6-27B.Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=16_000_000_000, + ) + captured: dict[str, object] = {} + + def fake_rank_models(models, hardware, **kwargs): + captured["quant_filter"] = kwargs.get("quant_filter") + return [ + CompatibilityResult( + model=selected, + gguf_variant=synthetic, + can_run=True, + vram_required_bytes=0, + vram_available_bytes=0, + quality_score=90.0, + ) + ] + + def fake_generate_chat_script(model, variant, context_length, cpu_only): + captured["model_id"] = model.id + captured["variant"] = variant + return "print('ok')" + + class Completed: + returncode = 0 + + def fake_run(cmd): + captured["cmd"] = cmd + return Completed() + + monkeypatch.setattr("shutil.which", lambda _: "/usr/bin/uv") + monkeypatch.setattr(cli_mod, "_load_models", lambda refresh: [selected, real_gguf]) + monkeypatch.setattr( + "whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8) + ) + monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {}) + monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models) + monkeypatch.setattr(cli_mod, "_generate_chat_script", fake_generate_chat_script) + monkeypatch.setattr("subprocess.run", fake_run) + + result = CliRunner().invoke(app, ["run", "--quant", "Q4_K_M"]) + + assert result.exit_code == 0 + assert captured["quant_filter"] == "Q4_K_M" + assert captured["model_id"] == "unsloth/Qwen3.6-27B-GGUF" + assert captured["variant"].filename == "q4.gguf" + assert "llama-cpp-python" in captured["cmd"] + assert "transformers" not in captured["cmd"] + + +def test_snippet_no_model_found(monkeypatch): + monkeypatch.setattr(cli_mod, "_load_models", lambda refresh: []) + runner = CliRunner() + result = runner.invoke(app, ["snippet", "nonexistent_model_xyz_999"]) + assert result.exit_code != 0 + assert "No model found" in result.stdout + + +def test_json_output_includes_benchmark_source_and_confidence(): + """display_json should include benchmark_source and benchmark_confidence.""" + import json as json_mod + from io import StringIO + + from rich.console import Console + + from whichllm.output.display import display_json + + model = ModelInfo( + id="test-org/Test-7B", + family_id="test-7b", + name="Test-7B", + parameter_count=7_000_000_000, + downloads=100, + likes=10, + ) + result = CompatibilityResult( + model=model, + gguf_variant=None, + can_run=True, + vram_required_bytes=8_000_000_000, + vram_available_bytes=24_000_000_000, + quality_score=55.0, + benchmark_status="estimated", + benchmark_source="line_interp", + benchmark_confidence=0.34, + ) + hw = HardwareInfo( + gpus=[], + cpu_name="Test CPU", + cpu_cores=8, + ram_budget_bytes=32 * 1024**3, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + budget_notes=["RAM budget: 32.0 GB"], + ) + + buf = StringIO() + import whichllm.output._console as console_mod + + orig_console = console_mod.console + console_mod.console = Console(file=buf, force_terminal=False) + try: + display_json([result], hw) + finally: + console_mod.console = orig_console + + data = json_mod.loads(buf.getvalue().strip()) + entry = data["models"][0] + assert data["hardware"]["ram_budget_bytes"] == 32 * 1024**3 + assert data["hardware"]["budget_notes"] == ["RAM budget: 32.0 GB"] + assert entry["artifact_repo_id"] is None + assert entry["artifact_filename"] is None + assert entry["benchmark_status"] == "estimated" + assert entry["benchmark_source"] == "line_interp" + assert entry["benchmark_confidence"] == 0.34 + + +def test_json_output_includes_resolved_artifact_fields(): + """display_json should expose the actual GGUF repo/file when resolved.""" + import json as json_mod + from io import StringIO + + from rich.console import Console + + from whichllm.output.display import display_json + + model = ModelInfo( + id="Qwen/Qwen3-4B-Thinking-2507", + family_id="qwen3-4b-thinking", + name="Qwen3-4B-Thinking-2507", + parameter_count=4_000_000_000, + ) + artifact = ModelInfo( + id="MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF", + family_id="qwen3-4b-thinking", + name="Qwen3-4B-Thinking-2507-GGUF", + parameter_count=4_000_000_000, + ) + result = CompatibilityResult( + model=model, + gguf_variant=GGUFVariant( + filename="Qwen3-4B-Thinking-2507.Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2_000_000_000, + ), + artifact_model=artifact, + artifact_variant=GGUFVariant( + filename="Qwen3-4B-Thinking-2507-Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2_000_000_000, + ), + can_run=True, + vram_required_bytes=3_000_000_000, + vram_available_bytes=8_000_000_000, + ) + hw = HardwareInfo( + gpus=[], + cpu_name="Test CPU", + cpu_cores=8, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + + buf = StringIO() + import whichllm.output._console as console_mod + + orig_console = console_mod.console + console_mod.console = Console(file=buf, force_terminal=False) + try: + display_json([result], hw) + finally: + console_mod.console = orig_console + + entry = json_mod.loads(buf.getvalue().strip())["models"][0] + assert entry["model_id"] == "Qwen/Qwen3-4B-Thinking-2507" + assert entry["artifact_repo_id"] == "MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF" + assert entry["artifact_filename"] == "Qwen3-4B-Thinking-2507-Q3_K_M.gguf" diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py new file mode 100644 index 0000000..6919fc8 --- /dev/null +++ b/tests/test_compatibility.py @@ -0,0 +1,502 @@ +"""Tests for compatibility checking.""" + +from whichllm.constants import _GiB +from whichllm.engine.compatibility import check_compatibility +from whichllm.hardware.memory import estimate_usable_ram +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo + + +def _make_model( + params: int = 7_000_000_000, context_length: int | None = None +) -> ModelInfo: + return ModelInfo( + id="test/model", + family_id="test/model", + name="model", + parameter_count=params, + context_length=context_length, + ) + + +def _make_variant(size: int = 4_000_000_000) -> GGUFVariant: + return GGUFVariant( + filename="model-Q4_K_M.gguf", quant_type="Q4_K_M", file_size_bytes=size + ) + + +def _make_hardware( + vram: int = 0, ram: int = 16 * 1024**3, disk: int = 100 * 1024**3, **gpu_kwargs +) -> HardwareInfo: + gpus = [] + if vram > 0: + gpus.append( + GPUInfo( + name="Test GPU", + vendor=gpu_kwargs.get("vendor", "nvidia"), + vram_bytes=vram, + compute_capability=gpu_kwargs.get("cc", (8, 6)), + memory_bandwidth_gbps=gpu_kwargs.get("bw", 500.0), + ) + ) + return HardwareInfo( + gpus=gpus, + cpu_name="Test CPU", + cpu_cores=8, + has_avx2=True, + ram_bytes=ram, + disk_free_bytes=disk, + os="linux", + ) + + +def test_full_gpu_fit(): + model = _make_model() + variant = _make_variant(4_000_000_000) + hw = _make_hardware(vram=24 * 1024**3) # 24GB VRAM + result = check_compatibility(model, variant, hw) + assert result.can_run is True + assert result.fit_type == "full_gpu" + + +def test_partial_offload(): + model = _make_model() + variant = _make_variant(20_000_000_000) # 20GB model + hw = _make_hardware(vram=8 * 1024**3, ram=64 * 1024**3) # 8GB VRAM, 64GB RAM + result = check_compatibility(model, variant, hw) + assert result.can_run is True + assert result.fit_type == "partial_offload" + assert 0.0 < result.offload_ratio < 1.0 + assert any("offload" in w.lower() for w in result.warnings) + + +def test_usable_vram_budget_can_turn_full_gpu_into_partial_offload(): + model = _make_model() + variant = _make_variant(7_000_000_000) + hw = _make_hardware(vram=8 * _GiB, ram=64 * _GiB) + hw.gpus[0].usable_vram_bytes = 6 * _GiB + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "partial_offload" + assert result.vram_available_bytes == 6 * _GiB + + +def test_ram_budget_limits_partial_offload_pool(): + model = _make_model() + variant = _make_variant(20_000_000_000) + hw = _make_hardware(vram=8 * _GiB, ram=64 * _GiB) + hw.ram_budget_bytes = 4 * _GiB + + result = check_compatibility(model, variant, hw) + + assert result.can_run is False + assert "Insufficient memory" in result.warnings[-1] + + +def test_ram_budget_caps_shared_memory_gpu_fit_pool(): + model = _make_model() + variant = _make_variant(12_000_000_000) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Apple M2", + vendor="apple", + vram_bytes=16 * _GiB, + usable_vram_bytes=15 * _GiB, + memory_bandwidth_gbps=100.0, + shared_memory=True, + ) + ], + cpu_name="Apple M2", + cpu_cores=8, + ram_bytes=16 * _GiB, + ram_budget_bytes=8 * _GiB, + disk_free_bytes=100 * _GiB, + os="darwin", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is False + assert result.vram_available_bytes == 8 * _GiB + + +def test_shared_memory_manual_vram_override_caps_available_gpu_memory(): + model = _make_model(8_000_000_000) + variant = _make_variant(4_000_000_000) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Intel UHD Graphics", + vendor="intel", + vram_bytes=1 * _GiB, + shared_memory=True, + vram_overridden=True, + ) + ], + cpu_name="Intel CPU", + cpu_cores=8, + ram_bytes=32 * _GiB, + disk_free_bytes=100 * _GiB, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.vram_available_bytes == 1 * _GiB + assert result.fit_type == "cpu_only" + + +def test_shared_memory_amd_apu_uses_system_memory_pool(): + model = _make_model(120_000_000_000) + variant = _make_variant(55_000_000_000) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="STRXLGEN", + vendor="amd", + vram_bytes=512 * 1024**2, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + ], + cpu_name="AMD Ryzen AI MAX+ 395", + cpu_cores=16, + ram_bytes=128 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "full_gpu" + assert result.vram_available_bytes == estimate_usable_ram(hw.ram_bytes) + assert not any("offload" in w.lower() for w in result.warnings) + assert not any("cpu only" in w.lower() for w in result.warnings) + + +def test_windows_shared_memory_amd_apu_does_not_emit_rocm_warning(): + model = _make_model(8_000_000_000) + variant = _make_variant(6_000_000_000) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="AMD Ryzen AI 9 HX 370 w/ Radeon 890M", + vendor="amd", + vram_bytes=0, + memory_bandwidth_gbps=120.0, + shared_memory=True, + ) + ], + cpu_name="AMD Ryzen AI 9 HX 370", + cpu_cores=12, + ram_bytes=16 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="windows", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "full_gpu" + assert result.vram_available_bytes == estimate_usable_ram(hw.ram_bytes) + assert not any("rocm" in w.lower() for w in result.warnings) + assert not any("offload" in w.lower() for w in result.warnings) + + +def test_shared_memory_igpu_is_not_summed_with_dedicated_gpu(): + model = _make_model(20_000_000_000) + variant = _make_variant(14 * 1024**3) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA GeForce RTX 4060", + vendor="nvidia", + vram_bytes=8 * 1024**3, + memory_bandwidth_gbps=272.0, + ), + GPUInfo( + name="Intel(R) Arc(TM) Graphics", + vendor="intel", + vram_bytes=0, + shared_memory=True, + ), + ], + cpu_name="Intel CPU", + cpu_cores=12, + ram_bytes=32 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="windows", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "partial_offload" + assert result.vram_available_bytes == 8 * 1024**3 + assert any("offloaded to CPU RAM" in w for w in result.warnings) + + +def test_homogeneous_multi_gpu_uses_conservative_fit_budget(): + model = _make_model(1_000_000_000) + variant = _make_variant(int(46 * _GiB)) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * _GiB, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ), + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * _GiB, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ), + ], + cpu_name="Test CPU", + cpu_cores=16, + ram_bytes=128 * _GiB, + disk_free_bytes=200 * _GiB, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "partial_offload" + assert result.uses_multi_gpu is True + assert result.vram_available_bytes == 48 * _GiB + assert result.multi_gpu_effective_vram_bytes is not None + assert result.multi_gpu_effective_vram_bytes < result.vram_available_bytes + assert any("conservative layer-split budget" in w for w in result.warnings) + + +def test_heterogeneous_multi_gpu_warns_about_split_assumptions(): + model = _make_model() + variant = _make_variant(20 * _GiB) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * _GiB, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ), + GPUInfo( + name="NVIDIA GeForce RTX 3060", + vendor="nvidia", + vram_bytes=12 * _GiB, + compute_capability=(8, 6), + memory_bandwidth_gbps=360.0, + ), + ], + cpu_name="Test CPU", + cpu_cores=16, + ram_bytes=64 * _GiB, + disk_free_bytes=200 * _GiB, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.uses_multi_gpu is True + assert result.multi_gpu_effective_vram_bytes is not None + assert result.multi_gpu_effective_vram_bytes < 36 * _GiB + assert any("Heterogeneous multi-GPU" in w for w in result.warnings) + + +def test_multiple_shared_memory_gpus_are_not_summed(): + model = _make_model(120_000_000_000) + variant = _make_variant(70 * _GiB) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Integrated GPU A", + vendor="amd", + vram_bytes=0, + memory_bandwidth_gbps=120.0, + shared_memory=True, + ), + GPUInfo( + name="Integrated GPU B", + vendor="intel", + vram_bytes=0, + shared_memory=True, + ), + ], + cpu_name="Test CPU", + cpu_cores=16, + ram_bytes=64 * _GiB, + disk_free_bytes=200 * _GiB, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert result.vram_available_bytes == estimate_usable_ram(hw.ram_bytes) + assert result.multi_gpu_effective_vram_bytes is None + assert result.fit_type == "cpu_only" + assert any("shared-memory GPUs are not pooled" in w for w in result.warnings) + + +def test_apple_silicon_does_not_double_count_unified_memory(): + """Apple Silicon uses unified memory: vram_bytes IS the system RAM. + The fit checker must not add a separate offload pool on top.""" + model = _make_model(70_000_000_000) + variant = _make_variant(40_000_000_000) # 40 GB model + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Apple M2 Max", + vendor="apple", + vram_bytes=32 * 1024**3, # 32 GB unified memory + memory_bandwidth_gbps=400.0, + shared_memory=True, + ) + ], + cpu_name="Apple M2 Max", + cpu_cores=12, + ram_bytes=32 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="darwin", + ) + + result = check_compatibility(model, variant, hw) + + # Model (40 GB) exceeds unified memory (32 GB). There is no separate + # CPU RAM pool to spill into, so this must NOT be partial_offload. + assert result.fit_type != "partial_offload", ( + "Apple Silicon should not get partial_offload — unified memory " + "cannot be double-counted as GPU VRAM + CPU RAM offload pool" + ) + assert not any("offloaded to CPU RAM" in w for w in result.warnings) + + +def test_apple_silicon_full_gpu_fit(): + """A model that fits within unified memory should be full_gpu.""" + model = _make_model(7_000_000_000) + variant = _make_variant(4_000_000_000) # 4 GB model + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Apple M4 Pro", + vendor="apple", + vram_bytes=24 * 1024**3, + memory_bandwidth_gbps=273.0, + shared_memory=True, + ) + ], + cpu_name="Apple M4 Pro", + cpu_cores=14, + ram_bytes=24 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="darwin", + ) + + result = check_compatibility(model, variant, hw) + + assert result.can_run is True + assert result.fit_type == "full_gpu" + assert not any("offload" in w.lower() for w in result.warnings) + + +def test_apple_silicon_vendor_guard_handles_legacy_shared_memory_false(): + """Even if a cached/older GPUInfo has shared_memory=False, the + vendor=='apple' guard should still prevent double-counting.""" + model = _make_model(70_000_000_000) + variant = _make_variant(40_000_000_000) # 40 GB model + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="Apple M2 Max", + vendor="apple", + vram_bytes=32 * 1024**3, + memory_bandwidth_gbps=400.0, + shared_memory=False, # legacy/cached object + ) + ], + cpu_name="Apple M2 Max", + cpu_cores=12, + ram_bytes=32 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="darwin", + ) + + result = check_compatibility(model, variant, hw) + + assert result.fit_type != "partial_offload", ( + "vendor='apple' guard must prevent double-counting even when " + "shared_memory=False (cached/older GPUInfo)" + ) + assert not any("offloaded to CPU RAM" in w for w in result.warnings) + + +def test_cpu_only(): + model = _make_model(1_000_000_000) + variant = _make_variant(600_000_000) + hw = _make_hardware(vram=0, ram=16 * 1024**3) # No GPU + result = check_compatibility(model, variant, hw) + assert result.can_run is True + assert result.fit_type == "cpu_only" + + +def test_insufficient_memory(): + model = _make_model(70_000_000_000) + variant = _make_variant(40_000_000_000) + hw = _make_hardware(vram=0, ram=8 * 1024**3) # Only 8GB RAM, no GPU + result = check_compatibility(model, variant, hw) + assert result.can_run is False + + +def test_low_compute_capability(): + model = _make_model() + variant = _make_variant(4_000_000_000) + hw = _make_hardware(vram=24 * 1024**3, cc=(4, 0)) # Very old GPU + result = check_compatibility(model, variant, hw) + assert result.can_run is True # Still runs, just with warning + assert any("compute capability" in w.lower() for w in result.warnings) + + +def test_insufficient_disk(): + model = _make_model() + variant = _make_variant(50_000_000_000) # 50GB file + hw = _make_hardware(vram=80 * 1024**3, disk=10 * 1024**3) # Only 10GB disk + result = check_compatibility(model, variant, hw) + assert result.can_run is False + assert any("disk" in w.lower() for w in result.warnings) + + +def test_context_fits_true_when_model_supports(): + model = _make_model(context_length=131072) + variant = _make_variant() + hw = _make_hardware(vram=24 * 1024**3) + result = check_compatibility(model, variant, hw, context_length=32768) + assert result.context_fits is True + + +def test_context_fits_false_when_model_too_small(): + model = _make_model(context_length=8192) + variant = _make_variant() + hw = _make_hardware(vram=24 * 1024**3) + result = check_compatibility(model, variant, hw, context_length=32768) + assert result.context_fits is False + assert any("max context" in w.lower() for w in result.warnings) + + +def test_context_fits_unknown_is_true(): + model = _make_model(context_length=None) + variant = _make_variant() + hw = _make_hardware(vram=24 * 1024**3) + result = check_compatibility(model, variant, hw, context_length=32768) + assert result.context_fits is True + assert not any("max context" in w.lower() for w in result.warnings) diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py new file mode 100644 index 0000000..0984e03 --- /dev/null +++ b/tests/test_fetcher.py @@ -0,0 +1,651 @@ +"""Tests for model metadata normalization in fetcher.""" + +import asyncio + +import httpx + +import whichllm.models.fetcher as fetcher +from whichllm.models.fetcher import ( + _extract_hf_eval_score, + _extract_published_at, + _hf_api_url, + _normalize_param_count, + _parse_model, + dicts_to_models, + models_to_dicts, +) +from whichllm.models.types import ModelInfo + + +def test_hf_api_url_uses_default_endpoint(monkeypatch): + monkeypatch.delenv("HF_ENDPOINT", raising=False) + + assert _hf_api_url("models") == "https://huggingface.co/api/models" + + +def test_hf_api_url_respects_hf_endpoint(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf-mirror.example") + + assert _hf_api_url("models/Qwen/Qwen3-8B") == ( + "https://hf-mirror.example/api/models/Qwen/Qwen3-8B" + ) + + +def test_hf_api_url_strips_trailing_slash(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf-mirror.example/") + + assert _hf_api_url("/models") == "https://hf-mirror.example/api/models" + + +def test_hf_api_url_rejects_empty_endpoint(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", " ") + + try: + _hf_api_url("models") + except ValueError as exc: + assert "HF_ENDPOINT must not be empty" in str(exc) + else: + raise AssertionError("expected empty HF_ENDPOINT to raise") + + +def test_hf_api_url_rejects_endpoint_without_scheme(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "hf-mirror.example") + + try: + _hf_api_url("models") + except ValueError as exc: + assert "HF_ENDPOINT must start with http:// or https://" in str(exc) + else: + raise AssertionError("expected invalid HF_ENDPOINT to raise") + + +def test_fetch_models_respects_hf_endpoint(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf-mirror.example") + urls: list[str] = [] + encodings: list[str] = [] + + async def fake_get_with_retries(client, url: str, **kwargs): + urls.append(url) + encodings.append(client.headers["accept-encoding"]) + request = httpx.Request("GET", url) + if "/models/" in url: + return httpx.Response(404, request=request) + return httpx.Response(200, json=[], request=request) + + monkeypatch.setattr(fetcher, "get_with_retries", fake_get_with_retries) + + models = asyncio.run(fetcher.fetch_models(limit=1, include_vision=False)) + + assert models == [] + assert urls + assert all(url.startswith("https://hf-mirror.example/api/") for url in urls) + assert "https://hf-mirror.example/api/models" in urls + assert not any(url.startswith("https://huggingface.co/api/") for url in urls) + assert set(encodings) == {"gzip, deflate"} + + +def test_fetch_model_published_at_respects_hf_endpoint(monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf-mirror.example") + urls: list[str] = [] + encodings: list[str] = [] + + async def fake_get(self, url: str, **kwargs): + urls.append(url) + encodings.append(self.headers["accept-encoding"]) + return httpx.Response( + 200, + json={"createdAt": "2026-06-22T00:00:00.000Z"}, + request=httpx.Request("GET", url), + ) + + monkeypatch.setattr(httpx.AsyncClient, "get", fake_get) + + result = asyncio.run(fetcher.fetch_model_published_at(["Qwen/Qwen3-8B"])) + + assert result == {"Qwen/Qwen3-8B": "2026-06-22T00:00:00.000Z"} + assert urls == ["https://hf-mirror.example/api/models/Qwen/Qwen3-8B"] + assert encodings == ["gzip, deflate"] + + +def test_normalize_param_count_for_quantized_repo_uses_size_hint(): + corrected = _normalize_param_count( + extracted=5_233_828_308, + model_id="ISTA-DASLab/gemma-3-27b-it-GPTQ-4b-128g", + base_model="google/gemma-3-27b-it", + ) + assert corrected == 27_000_000_000 + + +def test_normalize_param_count_keeps_reasonable_value(): + kept = _normalize_param_count( + extracted=11_765_788_416, + model_id="MaziyarPanahi/gemma-3-12b-it-GGUF", + base_model="google/gemma-3-12b-it", + ) + assert kept == 11_765_788_416 + + +def test_normalize_param_count_with_no_hint_keeps_original(): + kept = _normalize_param_count( + extracted=3_820_000_000, + model_id="microsoft/Phi-3-mini-4k-instruct-gguf", + base_model=None, + ) + assert kept == 3_820_000_000 + + +def test_dicts_to_models_normalizes_cached_parameter_count(): + models = dicts_to_models( + [ + { + "id": "ISTA-DASLab/gemma-3-27b-it-GPTQ-4b-128g", + "family_id": "gemma-3-27b", + "name": "gemma-3-27b-it-GPTQ-4b-128g", + "parameter_count": 5_233_828_308, + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + "base_model": "google/gemma-3-27b-it", + } + ] + ) + assert len(models) == 1 + assert models[0].parameter_count == 27_000_000_000 + + +def test_dicts_to_models_refreshes_cached_deepseek_v4_flash_counts(): + models = dicts_to_models( + [ + { + "id": "deepseek-ai/DeepSeek-V4-Flash", + "family_id": "deepseek-v4-flash", + "name": "DeepSeek-V4-Flash", + "parameter_count": 158_069_433_298, + "parameter_count_active": 10_000_000_000, + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count == 284_000_000_000 + assert models[0].parameter_count_active == 13_000_000_000 + assert models[0].is_moe is True + + +def test_dicts_to_models_uses_case_insensitive_curated_active_params(): + models = dicts_to_models( + [ + { + "id": "google/gemma-4-26B-A4B-it", + "family_id": "gemma-4-26b-a4b", + "name": "gemma-4-26B-A4B-it", + "parameter_count": 26_544_131_376, + "parameter_count_active": None, + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count_active == 3_800_000_000 + assert models[0].is_moe is True + + +def test_dicts_to_models_recovers_a3b_active_params_from_cached_qwen_model(): + models = dicts_to_models( + [ + { + "id": "Qwen/Qwen3.6-35B-A3B", + "family_id": "qwen3.6-35b-a3b", + "name": "Qwen3.6-35B-A3B", + "parameter_count": 35_951_822_704, + "parameter_count_active": None, + "architecture": "qwen3_5moe", + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count_active == 3_000_000_000 + assert models[0].is_moe is True + + +def test_dicts_to_models_recovers_a3b_active_params_from_base_model(): + models = dicts_to_models( + [ + { + "id": "local/Qwen36-GGUF", + "family_id": "qwen36-gguf", + "name": "Qwen36-GGUF", + "parameter_count": 34_660_610_688, + "parameter_count_active": None, + "architecture": "qwen35moe", + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + "base_model": "Qwen/Qwen3.6-35B-A3B", + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count_active == 3_000_000_000 + assert models[0].is_moe is True + + +def test_dicts_to_models_refreshes_stale_xiaomi_moe_cache_counts(): + models = dicts_to_models( + [ + { + "id": "XiaomiMiMo/MiMo-V2.5-Pro", + "family_id": "mimo-v2.5-pro", + "name": "MiMo-V2.5-Pro", + "parameter_count": 58_000_000_000, + "parameter_count_active": 11_000_000_000, + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count == 1_020_000_000_000 + assert models[0].parameter_count_active == 42_000_000_000 + assert models[0].is_moe is True + + +def test_dicts_to_models_recovers_missing_known_parameter_count(): + models = dicts_to_models( + [ + { + "id": "zai-org/GLM-5", + "family_id": "glm-5", + "name": "GLM-5", + "parameter_count": 0, + "parameter_count_active": None, + "downloads": 1, + "likes": 1, + "gguf_variants": [], + "benchmark_scores": {}, + } + ] + ) + + assert len(models) == 1 + assert models[0].parameter_count == 744_000_000_000 + assert models[0].parameter_count_active == 40_000_000_000 + assert models[0].is_moe is True + + +def test_parse_model_uses_current_glm5_and_xiaomi_active_counts(): + glm = _parse_model( + { + "id": "zai-org/GLM-5", + "config": {"architectures": ["GlmForCausalLM"]}, + "safetensors": {"total": 753_864_139_008}, + "siblings": [], + "cardData": {}, + } + ) + mimo = _parse_model( + { + "id": "XiaomiMiMo/MiMo-V2-Flash", + "config": {"architectures": ["LlamaForCausalLM"]}, + "safetensors": {"total": 309_785_318_400}, + "siblings": [], + "cardData": {}, + } + ) + + assert glm is not None + assert glm.parameter_count_active == 40_000_000_000 + assert mimo is not None + assert mimo.parameter_count_active == 15_000_000_000 + + +def test_parse_model_recovers_qwen36_a3b_active_params_from_name(): + parsed = _parse_model( + { + "id": "Qwen/Qwen3.6-35B-A3B", + "config": { + "architectures": ["Qwen3_5MoeForConditionalGeneration"], + "model_type": "qwen3_5_moe", + }, + "safetensors": {"total": 35_951_822_704}, + "siblings": [], + "cardData": {}, + } + ) + + assert parsed is not None + assert parsed.parameter_count == 35_951_822_704 + assert parsed.parameter_count_active == 3_000_000_000 + assert parsed.is_moe is True + + +def test_models_cache_roundtrip_keeps_published_at(): + models = [ + ModelInfo( + id="Qwen/Qwen3-8B-AWQ", + family_id="qwen3-8b", + name="Qwen3-8B-AWQ", + parameter_count=8_000_000_000, + published_at="2025-09-17T12:34:56.000Z", + downloads=123_456, + likes=789, + ) + ] + cached = models_to_dicts(models) + restored = dicts_to_models(cached) + assert len(restored) == 1 + assert restored[0].published_at == "2025-09-17T12:34:56.000Z" + assert restored[0].downloads == 123_456 + + +def test_extract_published_at_prefers_created_at(): + value = _extract_published_at( + { + "createdAt": "2025-01-01T00:00:00.000Z", + "lastModified": "2026-01-01T00:00:00.000Z", + } + ) + assert value == "2025-01-01T00:00:00.000Z" + + +def test_extract_published_at_falls_back_to_last_modified(): + value = _extract_published_at( + { + "lastModified": "2026-01-01T00:00:00.000Z", + } + ) + assert value == "2026-01-01T00:00:00.000Z" + + +def test_parse_model_keeps_split_gguf_as_single_variant(): + parsed = _parse_model( + { + "id": "org/Test-8B-GGUF", + "config": { + "architectures": ["LlamaForCausalLM"], + }, + "safetensors": {"total": 8_000_000_000}, + "siblings": [ + { + "rfilename": "model-Q4_K_M-00001-of-00002.gguf", + "size": 2_000_000_000, + }, + { + "rfilename": "model-Q4_K_M-00002-of-00002.gguf", + "size": 2_500_000_000, + }, + {"rfilename": "model-Q8_0.gguf", "size": 8_000_000_000}, + ], + "cardData": {}, + } + ) + assert parsed is not None + q4 = [v for v in parsed.gguf_variants if v.quant_type == "Q4_K_M"] + q8 = [v for v in parsed.gguf_variants if v.quant_type == "Q8_0"] + assert len(q4) == 1 + assert len(q8) == 1 + assert q4[0].file_size_bytes == 4_500_000_000 + + +def test_extract_hf_eval_score_uses_general_datasets_and_median(): + score = _extract_hf_eval_score( + { + "evalResults": [ + { + "filename": ".eval_results/mmlu-pro.yaml", + "data": {"dataset": {"id": "TIGER-Lab/MMLU-Pro"}, "value": 48.3}, + }, + { + "filename": ".eval_results/gsm8k.yaml", + "data": {"dataset": {"id": "openai/gsm8k"}, "value": 84.5}, + }, + { + "filename": ".eval_results/hle_medium_with_tools.yaml", + "data": { + "dataset": {"id": "cais/hle"}, + "value": 99.0, + "notes": "Reasoning: medium, With tools", + }, + }, + { + "filename": ".eval_results/swe_bench.yaml", + "data": { + "dataset": {"id": "SWE-bench/SWE-bench_Verified"}, + "value": 53.2, + }, + }, + ] + } + ) + # general対象(mmlu/gsm8k)のみ集計し、中央値を使う。 + assert score == 66.4 + + +def test_parse_model_extracts_hf_eval_benchmark_score(): + parsed = _parse_model( + { + "id": "meta-llama/Llama-3.1-8B-Instruct", + "config": {"architectures": ["LlamaForCausalLM"]}, + "safetensors": {"total": 8_000_000_000}, + "siblings": [], + "cardData": {}, + "evalResults": [ + { + "filename": ".eval_results/mmlu-pro.yaml", + "data": {"dataset": {"id": "TIGER-Lab/MMLU-Pro"}, "value": 48.3}, + }, + { + "filename": ".eval_results/gsm8k.yaml", + "data": {"dataset": {"id": "openai/gsm8k"}, "value": 84.5}, + }, + ], + } + ) + assert parsed is not None + assert parsed.benchmark_scores.get("hf_eval") == 66.4 + + +def test_deepseek_v4_flash_uses_model_card_counts_over_hf_tensor_metadata(): + """DeepSeek V4 Flash's mixed-precision HF tensor metadata reports a + smaller stored tensor count than the model-card total. Ranking and GGUF + synthesis must use the published model capacity instead.""" + + parsed = _parse_model( + { + "id": "deepseek-ai/DeepSeek-V4-Flash", + "config": { + "architectures": ["DeepseekV4ForCausalLM"], + "model_type": "deepseek_v4", + "quantization_config": {"quant_method": "fp8"}, + }, + "safetensors": { + "total": 158_069_433_298, + "parameters": { + "BF16": 1_415_259_264, + "F8_E8M0": 8_858_737_664, + "F8_E4M3": 6_023_020_544, + "I8": 141_733_920_768, + }, + }, + "siblings": [], + "cardData": {}, + } + ) + + assert parsed is not None + assert parsed.parameter_count == 284_000_000_000 + assert parsed.parameter_count_active == 13_000_000_000 + + +def test_parse_model_resolves_gemma3_sliding_window(): + parsed = _parse_model( + { + "id": "google/gemma-3-27b-it", + "config": { + "architectures": ["Gemma3ForConditionalGeneration"], + "model_type": "gemma3", + "sliding_window": 1024, + "sliding_window_pattern": 6, + "max_position_embeddings": 131072, + }, + "safetensors": {"total": 27_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window == 1024 + assert parsed.sliding_window_global_ratio == 1.0 / 6.0 + + +def test_parse_model_resolves_gpt_oss_sliding_window(): + parsed = _parse_model( + { + "id": "openai/gpt-oss-20b", + "config": { + "architectures": ["GptOssForCausalLM"], + "model_type": "gpt_oss", + "sliding_window": 128, + "max_position_embeddings": 131072, + }, + "safetensors": {"total": 20_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window == 128 + assert parsed.sliding_window_global_ratio == 0.5 + + +def test_parse_model_does_not_honor_mistral_sliding_window(): + """Mistral declares sliding_window in config but runtimes ignore it.""" + parsed = _parse_model( + { + "id": "mistralai/Mistral-7B-v0.1", + "config": { + "architectures": ["MistralForCausalLM"], + "model_type": "mistral", + "sliding_window": 4096, + "max_position_embeddings": 32768, + }, + "safetensors": {"total": 7_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window is None + assert parsed.sliding_window_global_ratio is None + + +def test_parse_model_respects_disabled_sliding_window(): + parsed = _parse_model( + { + "id": "google/gemma-3-4b-it", + "config": { + "architectures": ["Gemma3ForConditionalGeneration"], + "model_type": "gemma3", + "sliding_window": 1024, + "use_sliding_window": False, + }, + "safetensors": {"total": 4_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window is None + assert parsed.sliding_window_global_ratio is None + + +def test_parse_model_does_not_honor_gemma1_window(): + """Gemma-1 has no sliding window; the gemma2/gemma3 keys must not match it.""" + parsed = _parse_model( + { + "id": "google/gemma-7b", + "config": { + "architectures": ["GemmaForCausalLM"], + "model_type": "gemma", + }, + "safetensors": {"total": 7_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window is None + + +def test_parse_model_sliding_window_from_gguf_metadata(): + """GGUF-only repos: trust the authoritative GGUF architecture metadata.""" + parsed = _parse_model( + { + "id": "unsloth/gemma-3-12b-it-GGUF", + "config": {}, + "gguf": {"architecture": "gemma3"}, + "safetensors": {"total": 12_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window == 1024 + assert parsed.sliding_window_global_ratio == 1.0 / 6.0 + + +def test_parse_model_sliding_window_unset_without_metadata(): + """With neither HF config nor GGUF arch metadata, SWA stays unhonored. + + Detection is limited to authoritative metadata; a GGUF-only repo whose id + merely names an SWA family keeps the conservative full-context estimate + rather than being guessed at from the id. + """ + parsed = _parse_model( + { + "id": "TheBloke/gemma-2-9b-it-GGUF", + "config": {}, + "gguf": {}, + "safetensors": {"total": 9_000_000_000}, + "siblings": [], + "cardData": {}, + } + ) + assert parsed is not None + assert parsed.sliding_window is None + assert parsed.sliding_window_global_ratio is None + + +def test_models_cache_roundtrip_keeps_sliding_window(): + models = [ + ModelInfo( + id="google/gemma-3-27b-it", + family_id="gemma-3-27b", + name="gemma-3-27b-it", + parameter_count=27_000_000_000, + sliding_window=1024, + sliding_window_global_ratio=1.0 / 6.0, + ) + ] + restored = dicts_to_models(models_to_dicts(models)) + assert restored[0].sliding_window == 1024 + assert restored[0].sliding_window_global_ratio == 1.0 / 6.0 diff --git a/tests/test_gpu_db.py b/tests/test_gpu_db.py new file mode 100644 index 0000000..1063091 --- /dev/null +++ b/tests/test_gpu_db.py @@ -0,0 +1,118 @@ +"""Tests for dbgpu-backed bandwidth resolution on detected hardware. + +The detection path must never give a laptop/mobile card its desktop bandwidth +(issues #74, #61, #93). These tests pin the strict matching rules: curated +values win, the curated lookup is mobile-aware, dbgpu fills gaps, and an +unknown card resolves to None rather than a wrong guess. +""" + +from whichllm.hardware.gpu_db import ( + _normalize_detected_name, + _static_bandwidth, + resolve_detected_bandwidth, +) + +_GiB = 1024**3 + + +# --- normalization --------------------------------------------------------- + + +def test_normalize_strips_vendor_trademark_and_maps_laptop_to_mobile(): + assert ( + _normalize_detected_name("NVIDIA GeForce RTX 5090 Laptop GPU") + == "GeForce RTX 5090 Mobile" + ) + assert _normalize_detected_name("Intel(R) Arc(TM) A770 Graphics") == "Arc A770" + assert _normalize_detected_name("AMD Radeon RX 6750 XT") == "Radeon RX 6750 XT" + + +def test_normalize_empty_or_vendor_only(): + assert _normalize_detected_name("") == "" + assert _normalize_detected_name("AMD") == "" + + +def test_normalize_adds_space_to_vram_bin_suffix(): + # #98: drivers write "12GB", dbgpu writes "12 GB". + assert _normalize_detected_name("NVIDIA RTX A2000 12GB") == "RTX A2000 12 GB" + + +# --- curated lookup is mobile-aware --------------------------------------- + + +def test_static_bandwidth_desktop_key_does_not_claim_laptop_card(): + # "RTX 5090" desktop key (1792) must not match a Laptop driver name. + assert _static_bandwidth("NVIDIA GeForce RTX 5090 Laptop GPU") is None + # The desktop card itself still resolves from the curated table. + assert _static_bandwidth("NVIDIA GeForce RTX 5090") == 1792.0 + + +def test_static_bandwidth_keeps_curated_laptop_entry(): + # A curated key that is itself a laptop entry stays authoritative. + assert _static_bandwidth("NVIDIA RTX A3000 Laptop GPU") == 264.0 + + +# --- end-to-end resolution ------------------------------------------------- + + +def test_resolve_desktop_uses_curated_value(): + assert resolve_detected_bandwidth("NVIDIA GeForce RTX 5090") == 1792.0 + + +def test_resolve_laptop_5090_is_mobile_not_desktop(): + # The bug in #74: a laptop 5090 must not inherit the 1792 desktop value. + bw = resolve_detected_bandwidth("NVIDIA GeForce RTX 5090 Laptop GPU", 24 * _GiB) + assert bw is not None + assert bw < 1500.0 # mobile 5090 is ~896 GB/s, nowhere near desktop 1792 + + +def test_resolve_a3000_laptop_preserves_curated_264(): + assert resolve_detected_bandwidth("NVIDIA RTX A3000 Laptop GPU", 6 * _GiB) == 264.0 + + +def test_resolve_rx_6750_xt(): + # #61: originally None. The curated table now carries it (#68); dbgpu + # would agree (432 GB/s) if the curated entry ever went away. + bw = resolve_detected_bandwidth("AMD Radeon RX 6750 XT", 12 * _GiB) + assert bw == 432.0 + + +def test_resolve_variant_qualifier_is_preserved(): + # "RTX 4060 Ti" must not collapse to the non-Ti "RTX 4060". + bw = resolve_detected_bandwidth("NVIDIA GeForce RTX 4060 Ti", 16 * _GiB) + assert bw is not None + # 4060 (non-Ti) is 272 GB/s; the Ti is 288. Either way it must be a real, + # positive value and not desktop-flagship sized. + assert 200 < bw < 400 + + +def test_resolve_arc_pro_b70_uses_curated_value(): + assert resolve_detected_bandwidth("Intel(R) Arc(TM) Pro B70 Graphics") == 608.0 + assert resolve_detected_bandwidth("Battlemage G31 [Intel Graphics]") == 608.0 + + +def test_resolve_empty_name_returns_none(): + assert resolve_detected_bandwidth("") is None + assert resolve_detected_bandwidth("Some Totally Made Up GPU 9xZ") is None + + +def test_resolve_known_amd_desktop_from_curated_table(): + assert resolve_detected_bandwidth("AMD Radeon RX 9070 XT") == 640.0 + + +def test_resolve_a2000_12gb_vram_bin_from_dbgpu(): + # #98: "NVIDIA RTX A2000 12GB" reported BW: N/A. dbgpu names the bin + # "RTX A2000 12 GB" (288 GB/s); the normalizer must bridge the spacing. + assert resolve_detected_bandwidth("NVIDIA RTX A2000 12GB", 12 * _GiB) == 288.0 + + +def test_static_bandwidth_compound_lspci_name(): + # Ported from amd.py (#68): compound lspci names resolve via segment + # splitting, first matching segment wins, "RX " prefix retried for bare + # segments. + compound = "Navi 22 [Radeon RX 6700/6700 XT/6750 XT / 6800M/6850M XT]" + bw = _static_bandwidth(compound) + assert bw is not None + assert bw > 0 + # The same answer flows through the public resolver. + assert resolve_detected_bandwidth(compound, 12 * _GiB) == bw diff --git a/tests/test_gpu_simulator.py b/tests/test_gpu_simulator.py new file mode 100644 index 0000000..56864af --- /dev/null +++ b/tests/test_gpu_simulator.py @@ -0,0 +1,171 @@ +"""Tests for GPU simulator (--gpu flag) using dbgpu database.""" + +import pytest + +from whichllm.constants import _GiB +from whichllm.hardware.gpu_simulator import ( + create_synthetic_gpu, + create_synthetic_gpus, + parse_synthetic_gpu_specs, +) + + +class TestMultiGPUSpecParsing: + def test_comma_separated_gpu_specs(self): + assert parse_synthetic_gpu_specs(["RTX 4090, RTX 3090"]) == [ + "RTX 4090", + "RTX 3090", + ] + + def test_repeated_gpu_specs(self): + assert parse_synthetic_gpu_specs(["RTX 4090", "RTX 3090"]) == [ + "RTX 4090", + "RTX 3090", + ] + + def test_count_shorthand(self): + assert parse_synthetic_gpu_specs(["2x RTX 4090, 1x RTX 3090"]) == [ + "RTX 4090", + "RTX 4090", + "RTX 3090", + ] + + def test_empty_entry_raises(self): + with pytest.raises(ValueError, match="Empty GPU entry"): + parse_synthetic_gpu_specs(["RTX 4090,"]) + + def test_create_synthetic_gpus_expands_count(self): + gpus = create_synthetic_gpus(["2x RTX 4090"]) + assert len(gpus) == 2 + assert all(gpu.vendor == "nvidia" for gpu in gpus) + assert all(gpu.vram_bytes == 24 * _GiB for gpu in gpus) + + def test_multi_gpu_vram_override_is_rejected(self): + with pytest.raises(ValueError, match="exactly one simulated GPU"): + create_synthetic_gpus(["2x RTX 4090"], vram_override_gb=24) + + +class TestKnownGPULookup: + def test_nvidia_rtx_4090(self): + gpu = create_synthetic_gpu("RTX 4090") + assert gpu.vram_bytes == 24 * _GiB + assert gpu.vendor == "nvidia" + assert gpu.memory_bandwidth_gbps is not None + assert gpu.compute_capability is not None + assert gpu.compute_capability[0] >= 8 + assert "(simulated)" in gpu.name + + def test_nvidia_rtx_3060(self): + gpu = create_synthetic_gpu("RTX 3060") + assert "RTX 3060" in gpu.name + assert "Ti" not in gpu.name # Should NOT match RTX 3060 Ti + assert gpu.vendor == "nvidia" + assert gpu.compute_capability is not None + + def test_nvidia_rtx_3060_ti(self): + gpu = create_synthetic_gpu("RTX 3060 Ti") + assert "RTX 3060 Ti" in gpu.name + assert gpu.vram_bytes == 8 * _GiB + assert gpu.vendor == "nvidia" + + def test_amd_rx_7900_xtx(self): + gpu = create_synthetic_gpu("RX 7900 XTX") + assert gpu.vram_bytes == 24 * _GiB + assert gpu.vendor == "amd" + assert gpu.memory_bandwidth_gbps is not None + + def test_amd_strix_halo_with_vram_override(self): + gpu = create_synthetic_gpu("Radeon 8060S", vram_override_gb=96) + assert gpu.vram_bytes == 96 * _GiB + assert gpu.vendor == "amd" + assert gpu.shared_memory is True + assert gpu.memory_bandwidth_gbps == 256.0 + + def test_nvidia_gtx_1080(self): + gpu = create_synthetic_gpu("GTX 1080") + assert gpu.vram_bytes == 8 * _GiB + assert gpu.vendor == "nvidia" + assert gpu.compute_capability == (6, 1) + + def test_a100_80gb_alias(self): + gpu = create_synthetic_gpu("A100 80GB") + assert gpu.vram_bytes == 80 * _GiB + assert gpu.vendor == "nvidia" + assert "(simulated)" in gpu.name + + def test_h100_80gb_alias(self): + gpu = create_synthetic_gpu("H100 80GB") + assert gpu.vram_bytes == 80 * _GiB + assert gpu.vendor == "nvidia" + assert "(simulated)" in gpu.name + + def test_intel_arc_pro_b70_curated_spec(self): + gpu = create_synthetic_gpu("Arc Pro B70") + assert gpu.name == "Intel Arc Pro B70 (simulated)" + assert gpu.vram_bytes == 32 * _GiB + assert gpu.vendor == "intel" + assert gpu.memory_bandwidth_gbps == 608.0 + assert gpu.shared_memory is False + + +class TestAppleSiliconAliases: + @pytest.mark.parametrize( + "chip", + [ + "M1", + "M1 Max", + "M1 Ultra", + "M2", + "M2 Max", + "M2 Ultra", + "M3", + "M3 Max", + "M3 Ultra", + "M4", + "M4 Max", + "M4 Ultra", + ], + ) + def test_apple_prefixed_alias_matches_plain_chip_name(self, chip): + plain = create_synthetic_gpu(chip) + prefixed = create_synthetic_gpu(f"Apple {chip}") + + assert prefixed.name == plain.name + assert prefixed.vendor == "apple" + assert prefixed.vram_bytes == plain.vram_bytes + assert prefixed.memory_bandwidth_gbps == plain.memory_bandwidth_gbps + + @pytest.mark.parametrize("chip", ["M1", "M2 Max", "M3 Ultra", "M4 Pro"]) + def test_apple_silicon_has_shared_memory(self, chip): + gpu = create_synthetic_gpu(chip) + assert gpu.shared_memory is True + + +class TestVRAMOverride: + def test_override_known_gpu(self): + gpu = create_synthetic_gpu("RTX 4060 Ti", vram_override_gb=16) + assert gpu.vram_bytes == 16 * _GiB + assert gpu.memory_bandwidth_gbps is not None + + def test_override_unknown_gpu(self): + gpu = create_synthetic_gpu("Nonexistent GPU 9999", vram_override_gb=48) + assert gpu.vram_bytes == 48 * _GiB + assert "(simulated)" in gpu.name + + +class TestUnknownGPU: + def test_unknown_without_vram_raises(self): + with pytest.raises(ValueError, match="Unknown GPU"): + create_synthetic_gpu("Nonexistent GPU 9999") + + def test_unknown_with_vram_succeeds(self): + gpu = create_synthetic_gpu("Nonexistent GPU 9999", vram_override_gb=24) + assert gpu.vram_bytes == 24 * _GiB + + +class TestFuzzySearch: + def test_partial_name(self): + """dbgpu fuzzy search should find GPU from partial name.""" + gpu = create_synthetic_gpu("GTX 1080") + assert "1080" in gpu.name + assert gpu.vendor == "nvidia" diff --git a/tests/test_grouper.py b/tests/test_grouper.py new file mode 100644 index 0000000..ca3df07 --- /dev/null +++ b/tests/test_grouper.py @@ -0,0 +1,65 @@ +"""Tests for model grouping logic.""" + +from whichllm.models.grouper import _normalize_name, group_models +from whichllm.models.types import ModelInfo + + +def _make_model( + id: str, base_model: str | None = None, downloads: int = 100 +) -> ModelInfo: + return ModelInfo( + id=id, + family_id=id, + name=id.split("/")[-1], + parameter_count=7_000_000_000, + downloads=downloads, + base_model=base_model, + ) + + +def test_group_by_base_model(): + base = _make_model("meta/Llama-3-8B", downloads=1000) + gguf = _make_model( + "user/Llama-3-8B-GGUF", base_model="meta/Llama-3-8B", downloads=500 + ) + families = group_models([base, gguf]) + assert len(families) == 1 + assert families[0].base_model.id in ("meta/Llama-3-8B", "user/Llama-3-8B-GGUF") + + +def test_group_by_name_normalization(): + base = _make_model("org/model-v1", downloads=1000) + gguf = _make_model("org/model-v1-GGUF", downloads=200) + families = group_models([base, gguf]) + assert len(families) <= 2 # may or may not merge depending on normalization + + +def test_fp4_suffixes_normalize_to_base_family(): + # MXFP4 and NVFP4 derivatives must collapse onto the base family the same + # way the older quant suffixes do, instead of orphaning into their own. + base = _normalize_name("openai/gpt-oss-20b") + assert _normalize_name("openai/gpt-oss-20b-MXFP4") == base + assert _normalize_name("openai/gpt-oss-20b-NVFP4") == base + + +def test_ungrouped_models_separate(): + m1 = _make_model("org/alpha", downloads=100) + m2 = _make_model("org/beta", downloads=200) + families = group_models([m1, m2]) + assert len(families) == 2 + + +def test_empty_input(): + families = group_models([]) + assert families == [] + + +def test_family_id_set(): + base = _make_model("meta/Llama-3-8B", downloads=1000) + gguf = _make_model( + "user/Llama-3-8B-GGUF", base_model="meta/Llama-3-8B", downloads=500 + ) + families = group_models([base, gguf]) + for family in families: + assert family.family_id + assert family.base_model.family_id == family.family_id diff --git a/tests/test_gtx1650_variants.py b/tests/test_gtx1650_variants.py new file mode 100644 index 0000000..debcc20 --- /dev/null +++ b/tests/test_gtx1650_variants.py @@ -0,0 +1,171 @@ +"""GTX 1650 GDDR5 vs GDDR6 memory-variant disambiguation. + +The GTX 1650 shipped in two memory configurations the driver name and PCI +device id (0x1F82) cannot tell apart: the original GDDR5 (8 Gbps x 128-bit = +128 GB/s) and a later GDDR6 revision (12 Gbps x 128-bit = 192 GB/s). They are +resolved by the max memory clock reported at detection time. + +Bandwidth evidence (measured on a GDDR6 board, VBIOS 90.17.4D.00.1E): Qwen3-1.7B +Q4_K_M decodes at 75.4 tok/s clock-locked, matching the 192 GB/s estimate (~78) +and not the GDDR5 128 GB/s estimate (~52). +""" + +import subprocess + +import pytest + +from whichllm.constants import GPU_BANDWIDTH, GPU_MEMORY_CLOCK_VARIANTS +from whichllm.data.gpu import ( + GPU_MEMORY_CLOCK_VARIANTS as DATA_GPU_MEMORY_CLOCK_VARIANTS, +) +from whichllm.hardware import nvidia +from whichllm.hardware.gpu_db import resolve_detected_bandwidth +from whichllm.hardware.types import GPUInfo +from whichllm.engine.performance import estimate_tok_per_sec +from whichllm.models.types import GGUFVariant, ModelInfo + +_NAME = "NVIDIA GeForce GTX 1650" +GDDR6_CLOCK = 6001.0 # measured on the GDDR6 board +GDDR5_CLOCK = 4001.0 # typical GDDR5 1650 + + +def test_variant_table_present_and_reexported(): + assert "GTX 1650" in GPU_MEMORY_CLOCK_VARIANTS + # constants is a shim over whichllm.data.gpu. + assert GPU_MEMORY_CLOCK_VARIANTS is DATA_GPU_MEMORY_CLOCK_VARIANTS + # Highest threshold first, descending — required for first-match resolution. + thresholds = [t for t, _bw in GPU_MEMORY_CLOCK_VARIANTS["GTX 1650"]] + assert thresholds == sorted(thresholds, reverse=True) + + +def test_curated_default_is_gddr5(): + # The base key stays the conservative GDDR5 value for unknown-clock cases. + assert GPU_BANDWIDTH["GTX 1650"] == 128.0 + + +def test_gddr6_clock_resolves_to_192(): + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3, GDDR6_CLOCK) == 192.0 + + +def test_gddr5_clock_resolves_to_128(): + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3, GDDR5_CLOCK) == 128.0 + + +def test_unknown_clock_falls_back_to_curated_default(): + # No clock => identical to pre-change behaviour (GDDR5 default). + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3) == 128.0 + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3, None) == 128.0 + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3, 0.0) == 128.0 + + +@pytest.mark.parametrize( + "clock, expected", [(5499.0, 128.0), (5500.0, 192.0), (12000.0, 192.0)] +) +def test_threshold_boundary(clock, expected): + assert resolve_detected_bandwidth(_NAME, 4 * 1024**3, clock) == expected + + +def test_non_variant_card_ignores_memory_clock(): + # A single-memory-type card must resolve to its curated value regardless of + # any clock passed in (clock only disambiguates listed variants). + assert ( + resolve_detected_bandwidth("NVIDIA GeForce GTX 1660", 6 * 1024**3, 9999.0) + == 192.0 + ) + assert ( + resolve_detected_bandwidth("NVIDIA GeForce GTX 1660", 6 * 1024**3, 100.0) + == 192.0 + ) + + +def test_gtx1650_super_does_not_fall_through_to_base_1650(): + assert ( + resolve_detected_bandwidth("NVIDIA GeForce GTX 1650 SUPER", 4 * 1024**3) + == 192.0 + ) + assert ( + resolve_detected_bandwidth("NVIDIA GeForce GTX 1650 SUPER", 4 * 1024**3, 0.0) + == 192.0 + ) + + +def test_gddr6_estimate_scales_with_bandwidth_and_matches_measured(): + # Bandwidth flows linearly into the tok/s estimate; the GDDR6 estimate must + # exceed the GDDR5 one by the bandwidth ratio, and land near the measured + # 75.4 tok/s for Qwen3-1.7B Q4_K_M. + model = ModelInfo( + id="Qwen/Qwen3-1.7B", + family_id="Qwen/Qwen3-1.7B", + name="Qwen3-1.7B", + parameter_count=1_720_000_000, + ) + variant = GGUFVariant( + filename="Qwen3-1.7B-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=1_353_000_000, + ) + gpu6 = GPUInfo( + "NVIDIA GeForce GTX 1650", + "nvidia", + 4 * 1024**3, + compute_capability=(7, 5), + memory_bandwidth_gbps=192.0, + ) + gpu5 = GPUInfo( + "NVIDIA GeForce GTX 1650", + "nvidia", + 4 * 1024**3, + compute_capability=(7, 5), + memory_bandwidth_gbps=128.0, + ) + est6 = estimate_tok_per_sec(model, variant, gpu6) + est5 = estimate_tok_per_sec(model, variant, gpu5) + assert est6 > est5 + assert est6 / est5 == pytest.approx(192.0 / 128.0, rel=1e-3) + # Measured 75.4 tok/s; estimate should be within the "high"-confidence band. + assert 60.0 <= est6 <= 95.0 + + +# --- nvidia-smi detection-path tests (the plumbing that feeds the resolver) --- + + +def _smi_bw(monkeypatch, stdout: str) -> float | None: + monkeypatch.setattr(nvidia, "_run_smi_query", lambda fields: stdout) + gpus = nvidia._detect_nvidia_gpus_via_smi() + assert len(gpus) == 1 + return gpus[0].memory_bandwidth_gbps + + +def test_smi_gddr6_clock_resolves_192(monkeypatch): + assert _smi_bw(monkeypatch, "NVIDIA GeForce GTX 1650, 4096, 6001\n") == 192.0 + + +def test_smi_gddr5_clock_resolves_128(monkeypatch): + assert _smi_bw(monkeypatch, "NVIDIA GeForce GTX 1650, 4096, 4001\n") == 128.0 + + +def test_smi_na_clock_falls_back_to_curated(monkeypatch): + # Cards/drivers that don't report the clock emit "[N/A]" (still exit 0). + assert _smi_bw(monkeypatch, "NVIDIA GeForce GTX 1650, 4096, [N/A]\n") == 128.0 + + +def test_smi_3field_query_failure_retries_without_clock(monkeypatch): + # If clocks.max.memory makes the 3-field query fail, detection must retry the + # 2-field query rather than returning zero GPUs (regression guard). + def fake_query(fields: str) -> str: + if "clocks" in fields: + raise subprocess.CalledProcessError(6, "nvidia-smi") + return "NVIDIA GeForce GTX 1650, 4096\n" + + monkeypatch.setattr(nvidia, "_run_smi_query", fake_query) + gpus = nvidia._detect_nvidia_gpus_via_smi() + assert len(gpus) == 1 # not wiped out + assert gpus[0].memory_bandwidth_gbps == 128.0 # no clock => curated default + + +def test_smi_both_queries_fail_returns_empty(monkeypatch): + def always_fail(fields: str) -> str: + raise FileNotFoundError("nvidia-smi") + + monkeypatch.setattr(nvidia, "_run_smi_query", always_fail) + assert nvidia._detect_nvidia_gpus_via_smi() == [] diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100644 index 0000000..1210621 --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,63 @@ +import asyncio + +import httpx +import pytest + +from whichllm.models.benchmark_sources.chatbot_arena import fetch_arena_scores +from whichllm.models.http import get_with_retries + + +def test_get_with_retries_retries_429_then_returns_response(monkeypatch): + calls = 0 + sleeps: list[float] = [] + + async def fake_sleep(delay: float) -> None: + sleeps.append(delay) + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + if calls < 3: + return httpx.Response(429, request=request) + return httpx.Response(200, json={"ok": True}, request=request) + + async def run() -> httpx.Response: + monkeypatch.setattr("whichllm.models.http.asyncio.sleep", fake_sleep) + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + return await get_with_retries( + client, + "https://example.test/models", + base_delay=0.01, + jitter=0, + ) + + response = asyncio.run(run()) + + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert calls == 3 + assert sleeps == [0.01, 0.02] + + +def test_benchmark_source_retries_429_before_final_failure(monkeypatch): + calls = 0 + + async def fake_sleep(delay: float) -> None: + return None + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(429, request=request) + + async def run() -> None: + monkeypatch.setattr("whichllm.models.http.asyncio.sleep", fake_sleep) + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(httpx.HTTPStatusError): + await fetch_arena_scores(client) + + asyncio.run(run()) + + assert calls == 3 diff --git a/tests/test_intel_gpu.py b/tests/test_intel_gpu.py new file mode 100644 index 0000000..fae340c --- /dev/null +++ b/tests/test_intel_gpu.py @@ -0,0 +1,130 @@ +"""Tests for Linux Intel integrated GPU detection.""" + +from __future__ import annotations + +import subprocess +from io import StringIO + +from rich.console import Console + +from whichllm.hardware import intel +from whichllm.hardware.types import GPUInfo, HardwareInfo + + +def test_detect_intel_gpu_from_lspci(monkeypatch): + output = ( + '00:02.0 "VGA compatible controller" "Intel Corporation" ' + '"Alder Lake-P GT1 [UHD Graphics]" -r0c "Dell" "Device 0b19"\n' + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(intel.subprocess, "run", fake_run) + + gpus = intel.detect_intel_gpus() + + assert len(gpus) == 1 + assert gpus[0].vendor == "intel" + assert gpus[0].vram_bytes == 0 + assert "UHD Graphics" in gpus[0].name + + +def test_detect_intel_arc_pro_b70_from_battlemage_g31_lspci(monkeypatch): + output = ( + '12:00.0 "VGA compatible controller" "Intel Corporation" ' + '"Battlemage G31 [Intel Graphics]"\n' + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(intel.subprocess, "run", fake_run) + + gpus = intel.detect_intel_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "Battlemage G31 [Intel Graphics]" + assert gpus[0].vendor == "intel" + assert gpus[0].vram_bytes == 32 * 1024**3 + assert gpus[0].memory_bandwidth_gbps == 608.0 + assert gpus[0].shared_memory is False + + +def test_detect_intel_gpu_ignores_non_display_lspci(monkeypatch): + output = '00:00.0 "Host bridge" "Intel Corporation" "Device 4621"\n' + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(intel.subprocess, "run", fake_run) + monkeypatch.setattr(intel, "_detect_from_sysfs", lambda: []) + + assert intel.detect_intel_gpus() == [] + + +def test_detect_intel_gpu_from_sysfs_when_lspci_missing(monkeypatch, tmp_path): + card = tmp_path / "card0" / "device" + card.mkdir(parents=True) + (card / "vendor").write_text("0x8086\n") + (card / "uevent").write_text("PCI_SLOT_NAME=0000:00:02.0\n") + + monkeypatch.setattr(intel, "_detect_from_lspci", lambda: []) + original_sysfs = intel._detect_from_sysfs + monkeypatch.setattr(intel, "_detect_from_sysfs", lambda: original_sysfs(tmp_path)) + + gpus = intel.detect_intel_gpus() + + assert len(gpus) == 1 + assert gpus[0].vendor == "intel" + assert gpus[0].vram_bytes == 0 + assert gpus[0].name == "Intel Integrated Graphics" + + +def test_detect_intel_arc_pro_b70_from_sysfs_device_id(monkeypatch, tmp_path): + card = tmp_path / "card0" / "device" + card.mkdir(parents=True) + (card / "vendor").write_text("0x8086\n") + (card / "device").write_text("0xe223\n") + + monkeypatch.setattr(intel, "_detect_from_lspci", lambda: []) + original_sysfs = intel._detect_from_sysfs + monkeypatch.setattr(intel, "_detect_from_sysfs", lambda: original_sysfs(tmp_path)) + + gpus = intel.detect_intel_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "Battlemage G31 [Intel Graphics]" + assert gpus[0].vram_bytes == 32 * 1024**3 + assert gpus[0].memory_bandwidth_gbps == 608.0 + assert gpus[0].shared_memory is False + + +def test_display_intel_shared_memory_without_zero_kb(monkeypatch): + from whichllm.output import _console as console_mod + from whichllm.output import display as display_mod + + buf = StringIO() + monkeypatch.setattr(console_mod, "console", Console(file=buf, force_terminal=False)) + + display_mod.display_hardware( + HardwareInfo( + gpus=[ + GPUInfo( + name="Alder Lake-P GT1 [UHD Graphics]", + vendor="intel", + vram_bytes=0, + shared_memory=True, + ) + ], + cpu_name="CPU", + cpu_cores=8, + ram_bytes=16 * 1024**3, + disk_free_bytes=100 * 1024**3, + os="linux", + ) + ) + + output = buf.getvalue() + assert "shared memory" in output + assert "0 KB" not in output diff --git a/tests/test_kepler_gpus.py b/tests/test_kepler_gpus.py new file mode 100644 index 0000000..7db9739 --- /dev/null +++ b/tests/test_kepler_gpus.py @@ -0,0 +1,174 @@ +"""Tests for legacy Kepler GPU entries and their Vulkan-only handling. + +Kepler cards (compute capability 3.0/3.5) have no CUDA support in modern +llama.cpp, so whichllm carries bandwidth + compute-capability data for them +and flags them as Vulkan-only via ``VULKAN_ONLY_GPUS``. +""" + +import pytest + +from whichllm.constants import ( + GPU_BANDWIDTH, + NVIDIA_COMPUTE_CAPABILITY, + VULKAN_ONLY_GPUS, +) +from whichllm.data.gpu import ( + GPU_BANDWIDTH as DATA_GPU_BANDWIDTH, + VULKAN_ONLY_GPUS as DATA_VULKAN_ONLY_GPUS, +) +from whichllm.engine.compatibility import check_compatibility +from whichllm.hardware.nvidia import ( + _lookup_bandwidth, + _lookup_compute_capability, +) +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo + +# (GPU name, expected bandwidth GB/s, expected compute capability) +KEPLER_GPUS: list[tuple[str, float, tuple[int, int]]] = [ + ("Quadro K6000", 288.0, (3, 5)), + ("Quadro K5200", 192.3, (3, 5)), + ("Quadro K4200", 173.0, (3, 0)), + ("Quadro K2200", 80.0, (3, 0)), + ("Quadro K620", 29.0, (3, 0)), + ("Quadro K420", 14.4, (3, 0)), + ("GTX 780", 288.4, (3, 5)), + ("GTX 770", 224.3, (3, 0)), + ("GTX 760", 192.2, (3, 0)), +] + + +@pytest.mark.parametrize("name, bandwidth, _cc", KEPLER_GPUS) +def test_kepler_gpu_present_in_bandwidth(name, bandwidth, _cc): + assert name in GPU_BANDWIDTH + assert GPU_BANDWIDTH[name] == bandwidth + + +@pytest.mark.parametrize("name, _bandwidth, cc", KEPLER_GPUS) +def test_kepler_gpu_present_in_compute_capability(name, _bandwidth, cc): + assert name in NVIDIA_COMPUTE_CAPABILITY + assert NVIDIA_COMPUTE_CAPABILITY[name] == cc + + +@pytest.mark.parametrize("name, _bandwidth, cc", KEPLER_GPUS) +def test_kepler_compute_capability_is_kepler_era(name, _bandwidth, cc): + # Kepler is always compute capability 3.x. + assert cc[0] == 3 + + +@pytest.mark.parametrize("name, _bandwidth, _cc", KEPLER_GPUS) +def test_kepler_gpu_marked_vulkan_only(name, _bandwidth, _cc): + assert name in VULKAN_ONLY_GPUS + + +def test_vulkan_only_set_matches_kepler_list(): + expected = {name for name, _bw, _cc in KEPLER_GPUS} + assert set(VULKAN_ONLY_GPUS) == expected + + +def test_vulkan_only_gpus_is_immutable(): + assert isinstance(VULKAN_ONLY_GPUS, frozenset) + + +def test_constants_shim_reexports_data_module(): + # The constants module is a compatibility shim over whichllm.data.gpu. + assert GPU_BANDWIDTH is DATA_GPU_BANDWIDTH + assert VULKAN_ONLY_GPUS is DATA_VULKAN_ONLY_GPUS + + +def test_every_vulkan_only_gpu_has_bandwidth_and_compute_capability(): + # Each flagged card must be fully described so downstream lookups work. + for name in VULKAN_ONLY_GPUS: + assert name in GPU_BANDWIDTH + assert name in NVIDIA_COMPUTE_CAPABILITY + + +@pytest.mark.parametrize("name, bandwidth, cc", KEPLER_GPUS) +def test_nvidia_name_lookup_resolves_kepler_specs(name, bandwidth, cc): + # Detection matches the registry substrings against the full marketing + # name reported by the driver (e.g. "NVIDIA Quadro K4200"). + full_name = f"NVIDIA {name}" + assert _lookup_bandwidth(full_name) == bandwidth + assert _lookup_compute_capability(full_name) == cc + + +def test_k4200_lookup_not_shadowed_by_k420(): + # "Quadro K420" is a substring of "Quadro K4200"; the length-sorted lookup + # must still resolve the longer name to its own bandwidth. + assert _lookup_bandwidth("NVIDIA Quadro K4200") == 173.0 + assert _lookup_bandwidth("NVIDIA Quadro K420") == 14.4 + + +def _kepler_hardware(name: str, cc: tuple[int, int], bandwidth: float) -> HardwareInfo: + return HardwareInfo( + gpus=[ + GPUInfo( + name=f"NVIDIA {name}", + vendor="nvidia", + vram_bytes=4 * 1024**3, + compute_capability=cc, + memory_bandwidth_gbps=bandwidth, + ) + ], + cpu_name="Test CPU", + cpu_cores=8, + ram_bytes=16 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="linux", + ) + + +@pytest.mark.parametrize("name, bandwidth, cc", KEPLER_GPUS) +def test_compatibility_flags_kepler_as_vulkan_only(name, bandwidth, cc): + model = ModelInfo( + id="test/model", + family_id="test/model", + name="model", + parameter_count=3_000_000_000, + ) + variant = GGUFVariant( + filename="model-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=2_000_000_000, + ) + hw = _kepler_hardware(name, cc, bandwidth) + + result = check_compatibility(model, variant, hw) + + assert any("vulkan" in w.lower() for w in result.warnings), ( + f"{name} should be flagged as Vulkan-only" + ) + + +def test_compatibility_does_not_flag_modern_nvidia_as_vulkan_only(): + model = ModelInfo( + id="test/model", + family_id="test/model", + name="model", + parameter_count=3_000_000_000, + ) + variant = GGUFVariant( + filename="model-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=2_000_000_000, + ) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ) + ], + cpu_name="Test CPU", + cpu_cores=8, + ram_bytes=32 * 1024**3, + disk_free_bytes=200 * 1024**3, + os="linux", + ) + + result = check_compatibility(model, variant, hw) + + assert not any("vulkan" in w.lower() for w in result.warnings) diff --git a/tests/test_markdown_output.py b/tests/test_markdown_output.py new file mode 100644 index 0000000..13fae9c --- /dev/null +++ b/tests/test_markdown_output.py @@ -0,0 +1,167 @@ +"""Tests for pasteable Markdown ranking output.""" + +from io import StringIO + +from rich.console import Console + +from whichllm.engine.types import CompatibilityResult +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo +from whichllm.output.markdown import display_markdown + + +def _capture_markdown( + results: list[CompatibilityResult], + hardware: HardwareInfo, + *, + show_status: bool, + empty_message: str | None = None, +) -> str: + import whichllm.output._console as console_mod + + buf = StringIO() + orig_console = console_mod.console + console_mod.console = Console(file=buf, force_terminal=False) + try: + display_markdown( + results, + hardware, + show_status=show_status, + empty_message=empty_message, + ) + finally: + console_mod.console = orig_console + return buf.getvalue().strip() + + +def _hardware() -> HardwareInfo: + return HardwareInfo( + gpus=[ + GPUInfo( + name="RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + memory_bandwidth_gbps=1008.0, + ) + ], + cpu_name="Test CPU", + cpu_cores=16, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + + +def _result( + index: int, + *, + benchmark_status: str = "direct", + speed_confidence: str = "medium", +) -> CompatibilityResult: + model = ModelInfo( + id=f"org/Test-{index}|Model", + family_id=f"test-{index}", + name=f"Test-{index}", + parameter_count=7_000_000_000 + index, + downloads=1_500 * index, + likes=index, + license="apache-2.0", + published_at=f"2026-01-0{index}T00:00:00Z", + ) + return CompatibilityResult( + model=model, + gguf_variant=GGUFVariant( + filename=f"test-{index}.gguf", + quant_type="Q4_K_M", + file_size_bytes=4 * 1024**3, + ), + can_run=True, + vram_required_bytes=(4 + index) * 1024**3, + vram_available_bytes=24 * 1024**3, + estimated_tok_per_sec=10.0 * index, + speed_confidence=speed_confidence, + quality_score=80.0 - index, + fit_type="full_gpu", + benchmark_status=benchmark_status, + benchmark_source=benchmark_status, + benchmark_confidence=1.0, + ) + + +def test_display_markdown_runtime_table_top_three(): + output = _capture_markdown( + [ + _result(1, speed_confidence="medium"), + _result(2, benchmark_status="estimated", speed_confidence="low"), + _result(3, benchmark_status="none", speed_confidence="high"), + ], + _hardware(), + show_status=True, + ) + + assert output.startswith("## Recommended Models") + assert ( + "| # | Model | Params | Quant | Fit | VRAM | Speed | Published | Score | License |" + in output + ) + assert ( + "| 1 | org/Test-1\\|Model | 7.0B | Q4_K_M | Full GPU | 5.0 GB | 10.0 tok/s ~ | 2026-01-01 | 79.0 | apache-2.0 |" + in output + ) + assert "20.0 tok/s ?" in output + assert "78.0 ~" in output + assert "77.0 ?" in output + + +def test_display_markdown_details_table_uses_metadata_columns(): + output = _capture_markdown([_result(1)], _hardware(), show_status=False) + + assert ( + "| # | Model | Params | Quant | Published | Downloads | Score | License |" + in output + ) + assert "Fit | VRAM | Speed" not in output + assert ( + "| 1 | org/Test-1\\|Model | 7.0B | Q4_K_M | 2026-01-01 | 1.5K | 79.0 | apache-2.0 |" + in output + ) + + +def test_display_markdown_links_to_resolved_artifact_repo(): + result = _result(1) + result.model.id = "Qwen/Qwen3-4B-Thinking-2507" + result.gguf_variant = GGUFVariant( + filename="Qwen3-4B-Thinking-2507.Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2 * 1024**3, + ) + result.artifact_model = ModelInfo( + id="MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF", + family_id=result.model.family_id, + name="Qwen3-4B-Thinking-2507-GGUF", + parameter_count=result.model.parameter_count, + ) + result.artifact_variant = GGUFVariant( + filename="Qwen3-4B-Thinking-2507-Q3_K_M.gguf", + quant_type="Q3_K_M", + file_size_bytes=2 * 1024**3, + ) + + output = _capture_markdown([result], _hardware(), show_status=False) + + assert ( + "[Qwen/Qwen3-4B-Thinking-2507]" + "(https://huggingface.co/MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF)" in output + ) + assert "Q3_K_M" in output + + +def test_display_markdown_empty_results(): + output = _capture_markdown( + [], + _hardware(), + show_status=True, + empty_message="Nothing matched.", + ) + + assert output == "## Recommended Models\n\nNothing matched." diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..879acf9 --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,40 @@ +"""Tests for hardware.memory — estimate_usable_ram bounded-reserve formula.""" + +import pytest + +from whichllm.hardware.memory import estimate_usable_ram + +_GiB = 1024**3 + + +def _expected_usable(total: int) -> int: + reserve = int(total * 0.15) + reserve = max(4 * _GiB, min(reserve, 32 * _GiB)) + return total - reserve + + +@pytest.mark.parametrize( + "total_gb", + [16, 32, 64, 128, 1024], + ids=["16GB", "32GB", "64GB", "128GB", "1TB"], +) +def test_estimate_usable_ram(total_gb): + total = total_gb * _GiB + assert estimate_usable_ram(total) == _expected_usable(total) + + +def test_16gb_hits_min_reserve(): + total = 16 * _GiB + assert estimate_usable_ram(total) == total - 4 * _GiB + + +def test_1tb_hits_max_reserve(): + total = 1024 * _GiB + assert estimate_usable_ram(total) == total - 32 * _GiB + + +def test_midrange_uses_percentage(): + total = 64 * _GiB + expected_reserve = int(total * 0.15) + assert 4 * _GiB < expected_reserve < 32 * _GiB + assert estimate_usable_ram(total) == total - expected_reserve diff --git a/tests/test_nvidia_detection.py b/tests/test_nvidia_detection.py new file mode 100644 index 0000000..9a955e9 --- /dev/null +++ b/tests/test_nvidia_detection.py @@ -0,0 +1,194 @@ +import builtins +import subprocess +from types import SimpleNamespace +from unittest.mock import Mock + +from whichllm.hardware.nvidia import detect_nvidia_gpus + + +def test_nvidia_smi_fallback_when_pynvml_missing(monkeypatch): + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + raise ImportError + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + return SimpleNamespace(stdout="NVIDIA GeForce RTX 5060 Ti, 16303\n") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA GeForce RTX 5060 Ti" + assert gpus[0].vendor == "nvidia" + assert gpus[0].vram_bytes == 16303 * 1024**2 + + +def test_nvidia_smi_fallback_applies_rtx_a3000_laptop_catalog(monkeypatch): + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + raise ImportError + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + return SimpleNamespace(stdout="NVIDIA RTX A3000 Laptop GPU, 6144 MiB\n") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA RTX A3000 Laptop GPU" + assert gpus[0].compute_capability == (8, 6) + assert gpus[0].memory_bandwidth_gbps == 264.0 + + +def test_nvidia_smi_fallback_resolves_laptop_5090_via_dbgpu(monkeypatch): + # Regression for #74: a laptop RTX 5090 must not inherit the desktop + # 1792 GB/s bandwidth. dbgpu carries "GeForce RTX 5090 Mobile" (~896). + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + raise ImportError + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + return SimpleNamespace(stdout="NVIDIA GeForce RTX 5090 Laptop GPU, 24564 MiB\n") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA GeForce RTX 5090 Laptop GPU" + bw = gpus[0].memory_bandwidth_gbps + assert bw is not None + assert bw < 1500.0 # mobile ~896, not the 1792 desktop value + + +def test_nvidia_smi_fallback_when_nvml_init_fails(monkeypatch): + class FakeNVMLError(Exception): + pass + + fake_pynvml = SimpleNamespace( + NVMLError=FakeNVMLError, + nvmlInit=Mock(side_effect=FakeNVMLError("NVML unavailable")), + ) + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + return fake_pynvml + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + return SimpleNamespace(stdout="NVIDIA DGX Spark, 128000 MiB\n") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA DGX Spark" + assert gpus[0].vendor == "nvidia" + assert gpus[0].vram_bytes == 128000 * 1024**2 + assert gpus[0].shared_memory is True + + +def test_nvidia_smi_fallback_detects_gb10_with_na_memory(monkeypatch): + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + raise ImportError + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + return SimpleNamespace(stdout="NVIDIA GB10, [N/A]\n") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr( + "whichllm.hardware.memory.detect_ram_bytes", lambda: 128 * 1024**3 + ) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA GB10" + assert gpus[0].vendor == "nvidia" + assert gpus[0].vram_bytes == 128 * 1024**3 + assert gpus[0].shared_memory is True + assert gpus[0].compute_capability == (12, 1) + assert gpus[0].memory_bandwidth_gbps == 273.0 + + +def test_nvml_detects_gb10_when_memory_info_is_unavailable(monkeypatch): + class FakeNVMLError(Exception): + pass + + handle = object() + fake_pynvml = SimpleNamespace( + NVMLError=FakeNVMLError, + nvmlInit=Mock(), + nvmlDeviceGetCount=Mock(return_value=1), + nvmlSystemGetDriverVersion=Mock(return_value=b"580.142"), + nvmlSystemGetCudaDriverVersion_v2=Mock(return_value=12010), + nvmlDeviceGetHandleByIndex=Mock(return_value=handle), + nvmlDeviceGetName=Mock(return_value="NVIDIA GB10"), + nvmlDeviceGetMemoryInfo=Mock(side_effect=FakeNVMLError("not supported")), + nvmlShutdown=Mock(), + ) + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + return fake_pynvml + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + raise AssertionError("nvidia-smi fallback should not be used") + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr( + "whichllm.hardware.memory.detect_ram_bytes", lambda: 128 * 1024**3 + ) + + gpus = detect_nvidia_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "NVIDIA GB10" + assert gpus[0].vendor == "nvidia" + assert gpus[0].vram_bytes == 128 * 1024**3 + assert gpus[0].shared_memory is True + assert gpus[0].compute_capability == (12, 1) + assert gpus[0].cuda_version == "12.1" + assert gpus[0].memory_bandwidth_gbps == 273.0 + + +def test_nvidia_smi_fallback_returns_empty_on_command_failure(monkeypatch): + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "pynvml": + raise ImportError + return real_import(name, *args, **kwargs) + + def fake_run(*args, **kwargs): + raise FileNotFoundError + + monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(subprocess, "run", fake_run) + + assert detect_nvidia_gpus() == [] diff --git a/tests/test_output_formatting.py b/tests/test_output_formatting.py new file mode 100644 index 0000000..36a916f --- /dev/null +++ b/tests/test_output_formatting.py @@ -0,0 +1,32 @@ +"""Tests for output formatting helpers.""" + +from whichllm.engine.types import CompatibilityResult +from whichllm.models.types import ModelInfo +from whichllm.output.formatting import _format_speed + + +def _result(speed: float, confidence: str) -> CompatibilityResult: + return CompatibilityResult( + model=ModelInfo( + id="org/model", + family_id="org/model", + name="model", + parameter_count=1_000_000_000, + ), + gguf_variant=None, + can_run=True, + vram_required_bytes=0, + vram_available_bytes=0, + estimated_tok_per_sec=speed, + speed_confidence=confidence, + ) + + +def test_format_speed_colors_by_runtime_speed_not_confidence(): + assert _format_speed(_result(2.5, "medium")) == "[red]2.5 tok/s ~[/red]" + assert _format_speed(_result(6.0, "low")) == "[yellow]6.0 tok/s ?[/yellow]" + assert _format_speed(_result(12.0, "medium")) == "[green]12.0 tok/s ~[/green]" + assert ( + _format_speed(_result(30.0, "low")) + == "[bright_green]30.0 tok/s ?[/bright_green]" + ) diff --git a/tests/test_p1_p3_regressions.py b/tests/test_p1_p3_regressions.py new file mode 100644 index 0000000..aa99b28 --- /dev/null +++ b/tests/test_p1_p3_regressions.py @@ -0,0 +1,513 @@ +"""Regression tests for the Phase 1-3 ranking-quality fixes. + +Each test reproduces a specific bug class that the original whichllm 0.5.0 +ranker exhibited on 2026-05-14: + +- Q1_0 / Q2_0 derivatives must not surface as full GPU candidates by + default (P1: extreme-quant exclusion). +- Self-reported (hf_eval) values must not beat independent leaderboard + evidence (P3-1: self_reported tier). +- CI / research orgs (gpt2, opt-125m, tiny-Qwen2ForCausalLM) must never + occupy a ranking slot (P2-2: excluded orgs). +- Newer generation in the same family should beat the older one when + benchmark data is unavailable or weak (P2-1: lineage bonus). +- Heretic / abliterated / uncensored derivatives should rank below their + clean base/converter counterparts (P2-3: derivative penalty). +""" + +from __future__ import annotations + +from whichllm.engine.ranker import ( + _SOURCE_WEIGHTS, + _derivative_name_penalty, + _generation_bonus, + _is_excluded_model, + rank_models, +) +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo + + +def _hw( + vram_gb: int = 24, + bandwidth_gbps: float = 1000.0, + vendor: str = "nvidia", + os_name: str = "linux", + with_gpu: bool = True, +) -> HardwareInfo: + gpus = [] + if with_gpu: + gpus = [ + GPUInfo( + name="Test GPU", + vendor=vendor, + vram_bytes=vram_gb * 1024**3, + compute_capability=(8, 9) if vendor == "nvidia" else None, + memory_bandwidth_gbps=bandwidth_gbps, + ) + ] + return HardwareInfo( + gpus=gpus, + cpu_name="Test CPU", + cpu_cores=8, + has_avx2=True, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os=os_name, + ) + + +def _gguf(quant: str, size_gb: float) -> GGUFVariant: + return GGUFVariant( + filename=f"model-{quant}.gguf", + quant_type=quant, + file_size_bytes=int(size_gb * 1e9), + ) + + +# ---------------------------------------------------------------------- P1 + + +def test_p1_q1_0_only_repo_is_severely_penalized_when_no_quant_filter(): + """An 8B repo that only ships Q1_0 may still appear as a fallback (since + it's the only thing the repo offers) but its quality score must be + crushed by the combined Q1_0 (-55%) + self_reported (-45%) penalties so + it cannot beat a normal Q4_K_M alternative.""" + bonsai = ModelInfo( + id="prism-ml/Bonsai-8B-gguf", + family_id="bonsai-8b", + name="Bonsai-8B", + parameter_count=8_000_000_000, + downloads=48_000, + likes=0, + gguf_variants=[_gguf("Q1_0", 4.6)], + benchmark_scores={"hf_eval": 88.0}, + ) + normal = ModelInfo( + id="bartowski/Qwen3-8B-GGUF", + family_id="qwen3-8b", + name="Qwen3-8B", + parameter_count=8_000_000_000, + downloads=58_000, + likes=20, + base_model="Qwen/Qwen3-8B", + gguf_variants=[_gguf("Q4_K_M", 4.7)], + ) + results = rank_models( + [bonsai, normal], + _hw(), + top_n=5, + benchmark_scores={"Qwen/Qwen3-8B-Instruct": 65.0}, + ) + ids = [r.model.id for r in results] + assert "bartowski/Qwen3-8B-GGUF" in ids + # The Q1_0 + self_reported combo must end up below the normal Q4_K_M. + sc_bonsai = next( + (r.quality_score for r in results if r.model.id == "prism-ml/Bonsai-8B-gguf"), + 0.0, + ) + sc_normal = next( + (r.quality_score for r in results if r.model.id == "bartowski/Qwen3-8B-GGUF"), + 0.0, + ) + assert sc_normal > sc_bonsai + # And the absolute Bonsai score must be in the "obviously broken" band. + assert sc_bonsai < 25 + + +def test_p1_q1_0_returned_when_explicitly_requested_via_quant_filter(): + """Users can still ask for Q1_0 with --quant Q1_0 when they really mean it.""" + model = ModelInfo( + id="prism-ml/Bonsai-8B-gguf", + family_id="bonsai-8b", + name="Bonsai-8B", + parameter_count=8_000_000_000, + downloads=48_000, + likes=0, + gguf_variants=[_gguf("Q1_0", 4.6)], + ) + results = rank_models( + [model], _hw(), top_n=5, benchmark_scores={}, quant_filter="Q1_0" + ) + assert len(results) == 1 + assert results[0].gguf_variant is not None + assert results[0].gguf_variant.quant_type == "Q1_0" + + +def test_p1_q1_q2_quality_penalty_is_severe(): + """Sub-2-bit quants must carry 40-60% quality penalty, not the + legacy 5% fallback. We assert the constant directly to lock this in.""" + from whichllm.constants import QUANT_QUALITY_PENALTY + + assert QUANT_QUALITY_PENALTY["Q1_0"] >= 0.50 + assert QUANT_QUALITY_PENALTY["Q2_0"] >= 0.40 + assert QUANT_QUALITY_PENALTY["TQ1_0"] >= 0.50 + assert QUANT_QUALITY_PENALTY["IQ1_S"] >= 0.50 + assert QUANT_QUALITY_PENALTY["IQ1_M"] >= 0.45 + assert QUANT_QUALITY_PENALTY["IQ2_M"] >= 0.25 + + +# ---------------------------------------------------------------------- P2-2 + + +def test_p2_excluded_orgs_never_rank(): + """gpt2 / opt-125m / TRL fixtures must be skipped regardless of DL.""" + gpt2 = ModelInfo( + id="openai-community/gpt2", + family_id="gpt2", + name="gpt2", + parameter_count=124_000_000, + downloads=16_000_000, + likes=1000, + ) + opt = ModelInfo( + id="facebook/opt-125m", + family_id="opt-125m", + name="opt-125m", + parameter_count=125_000_000, + downloads=9_000_000, + ) + tiny = ModelInfo( + id="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5", + family_id="tiny-q2", + name="tiny", + parameter_count=1_000_000, + downloads=6_000_000, + ) + # also a normal model so the result list is non-empty + real = ModelInfo( + id="bartowski/gemma-3-12b-it-GGUF", + family_id="gemma-3-12b", + name="gemma-3-12b", + parameter_count=12_200_000_000, + downloads=70_000, + likes=10, + gguf_variants=[_gguf("Q4_K_M", 7.0)], + ) + results = rank_models([gpt2, opt, tiny, real], _hw(), top_n=10, benchmark_scores={}) + ids = [r.model.id for r in results] + assert "openai-community/gpt2" not in ids + assert "facebook/opt-125m" not in ids + assert "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5" not in ids + assert "bartowski/gemma-3-12b-it-GGUF" in ids + + +def test_p2_is_excluded_model_helper(): + assert _is_excluded_model("openai-community/gpt2") + assert _is_excluded_model("facebook/opt-125m") + assert _is_excluded_model("EleutherAI/pythia-70m-deduped") + assert _is_excluded_model("trl-internal-testing/tiny-Qwen2ForCausalLM-2.5") + assert _is_excluded_model("hmellor/tiny-random-LlamaForCausalLM") + assert not _is_excluded_model("Qwen/Qwen3-8B") + assert not _is_excluded_model("meta-llama/Llama-4-Maverick-17B-128E-Instruct") + + +# ---------------------------------------------------------------------- P2-1 + + +def test_p2_generation_bonus_newer_wins_over_legacy_in_same_family(): + new = _generation_bonus("Qwen/Qwen3.6-27B") + mid = _generation_bonus("Qwen/Qwen3-8B") + old = _generation_bonus("Qwen/Qwen2.5-7B-Instruct") + ancient = _generation_bonus("Qwen/Qwen-7B") + assert new > mid > old > ancient + + +def test_p2_lineage_covers_llama_deepseek_gemma_phi(): + assert _generation_bonus("meta-llama/Llama-4-Scout") > _generation_bonus( + "meta-llama/Llama-3.1-8B-Instruct" + ) + assert _generation_bonus("deepseek-ai/DeepSeek-V4-Pro") > _generation_bonus( + "deepseek-ai/DeepSeek-V2.5" + ) + assert _generation_bonus("google/gemma-4-31b-it") > _generation_bonus( + "google/gemma-2-27b-it" + ) + assert _generation_bonus("microsoft/phi-4") > _generation_bonus( + "microsoft/Phi-3-mini-4k-instruct" + ) + + +def test_p2_lineage_covers_t5_variants_without_gemma_collision(): + assert _generation_bonus("google/t5gemma-4b") > _generation_bonus( + "google/flan-t5-xl" + ) + assert _generation_bonus("google/t5-gemma-4b") == _generation_bonus( + "google/t5gemma-4b" + ) + assert _generation_bonus("google/t5_gemma-4b") == _generation_bonus( + "google/t5gemma-4b" + ) + assert _generation_bonus("openai/gpt5-test") == 0.0 + + +def test_p2_unknown_family_gets_zero_bonus(): + assert _generation_bonus("random-org/random-model-7b") == 0.0 + + +# ---------------------------------------------------------------------- P2-3 + + +def test_p2_derivative_penalty_for_heretic_uncensored(): + assert ( + _derivative_name_penalty("diffusionmodels1254ani/gemma-3-12b-it-heretic-v2") < 0 + ) + assert ( + _derivative_name_penalty( + "Youssofal/Qwen3.6-27B-Abliterated-Heretic-Uncensored-GGUF" + ) + < 0 + ) + assert _derivative_name_penalty("OBLITERATUS/gemma-4-E4B-it-OBLITERATED") < 0 + assert _derivative_name_penalty("bartowski/Qwen3-32B-GGUF") == 0.0 + + +# ---------------------------------------------------------------------- P3-1 + + +def test_p3_self_reported_evidence_does_not_outrank_direct_leaderboard(): + """A self-reported eval (e.g. prism-ml/Bonsai claiming 91) must NOT + outrank an independent-leaderboard hit on a comparable model.""" + self_reported = ModelInfo( + id="prism-ml/Self-Reported-8B", + family_id="self-reported-8b", + name="Self-Reported", + parameter_count=8_000_000_000, + downloads=20_000, + likes=10, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + benchmark_scores={"hf_eval": 91.0}, + ) + direct_hit = ModelInfo( + id="trusted-org/Real-Bench-8B", + family_id="real-bench-8b", + name="Real-Bench", + parameter_count=8_000_000_000, + downloads=50_000, + likes=200, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + ) + results = rank_models( + [self_reported, direct_hit], + _hw(), + top_n=5, + benchmark_scores={"trusted-org/Real-Bench-8B": 70.0}, + ) + assert len(results) == 2 + assert results[0].model.id == "trusted-org/Real-Bench-8B" + assert results[0].benchmark_status == "direct" + assert results[1].benchmark_status == "self_reported" + + +def test_p3_self_reported_outranks_only_when_there_is_nothing_else(): + """When no other evidence exists, self_reported should still produce a + score so the candidate isn't filtered out — just at a lower weight.""" + only_self_reported = ModelInfo( + id="some-org/Only-Self-Reported-8B", + family_id="only-sr-8b", + name="Only-SR", + parameter_count=8_000_000_000, + downloads=20_000, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + benchmark_scores={"hf_eval": 90.0}, + ) + results = rank_models( + [only_self_reported], + _hw(), + top_n=5, + benchmark_scores={}, + ) + assert len(results) == 1 + assert results[0].benchmark_status == "self_reported" + # Score should be positive but well below what a 90 direct-leaderboard + # would have produced (which would push past 60 with this size class). + assert 0 < results[0].quality_score < 60 + + +def test_p3_source_weights_ordering(): + """Direct must weight the most, self_reported the least among + benchmark-producing sources.""" + assert _SOURCE_WEIGHTS["direct"] > _SOURCE_WEIGHTS["base_model"] + assert _SOURCE_WEIGHTS["base_model"] > _SOURCE_WEIGHTS["variant"] + assert _SOURCE_WEIGHTS["variant"] > _SOURCE_WEIGHTS["line_interp"] + assert _SOURCE_WEIGHTS["line_interp"] > _SOURCE_WEIGHTS["self_reported"] + assert _SOURCE_WEIGHTS["self_reported"] > 0 + assert _SOURCE_WEIGHTS["none"] == 0.0 + + +def test_p3_strict_evidence_filter_excludes_self_reported(): + """`--evidence strict` should keep only direct hits, dropping + self_reported (the exact bug that surfaced 5 prism-ml/squ11z1 + models as 'direct') as well as inherited evidence.""" + self_reported = ModelInfo( + id="some-org/Self-Reported-8B", + family_id="sr-8b", + name="SR", + parameter_count=8_000_000_000, + downloads=20_000, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + benchmark_scores={"hf_eval": 95.0}, + ) + direct_hit = ModelInfo( + id="trusted-org/Real-Bench-8B", + family_id="real-bench-8b", + name="Real-Bench", + parameter_count=8_000_000_000, + downloads=50_000, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + ) + results = rank_models( + [self_reported, direct_hit], + _hw(), + top_n=5, + benchmark_scores={"trusted-org/Real-Bench-8B": 70.0}, + evidence_filter="strict", + ) + ids = [r.model.id for r in results] + assert ids == ["trusted-org/Real-Bench-8B"] + + +# -------------------- Rating-push regressions (Phase 2) -------------------- + + +def test_official_org_safetensors_gets_q4km_synthesis(): + """A Qwen/Meta/Google safetensors-only repo should produce realistic + Q4_K_M candidates rather than being scored as bf16 (which forces + partial_offload on consumer GPUs).""" + model = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3.6-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + downloads=50_000, + gguf_variants=[], # safetensors only — synthesis should kick in + ) + results = rank_models( + [model], + _hw(vram_gb=24), + top_n=1, + benchmark_scores={"Qwen/Qwen3.6-27B": 84.0}, + ) + assert results + chosen = results[0] + assert chosen.fit_type == "full_gpu" + assert chosen.gguf_variant is not None + # Synthesis offers a range of K-quants — any of them is acceptable so + # long as a GGUF candidate is constructed (i.e. the safetensors-only + # repo is rankable at a realistic quant). + assert chosen.gguf_variant.quant_type in { + "Q3_K_M", + "Q4_K_M", + "Q5_K_M", + "Q6_K", + "Q8_0", + } + + +def test_prequantized_repo_skips_synthesis(): + """An -AWQ / -GPTQ repo must not get synthetic Q4_K_M candidates — those + repos can't actually serve a GGUF variant.""" + model = ModelInfo( + id="Qwen/Qwen2.5-14B-Instruct-AWQ", + family_id="qwen2.5-14b-awq", + name="Qwen2.5-14B-Instruct-AWQ", + parameter_count=14_000_000_000, + downloads=10_000, + ) + q4_filtered = rank_models( + [model], + _hw(vram_gb=24), + top_n=5, + benchmark_scores={"Qwen/Qwen2.5-14B-Instruct-AWQ": 70.0}, + quant_filter="Q4_K_M", + ) + assert q4_filtered == [] + + +def test_newer_generation_beats_older_at_same_size(): + """With the realigned AA/LB normalizers and stronger generation lineage + bonus, a current-gen 8B model should outrank a frozen-OLLB-favored + previous-gen 7B.""" + new_gen = ModelInfo( + id="Qwen/Qwen3-8B", + family_id="qwen3-8b", + name="Qwen3-8B", + parameter_count=8_000_000_000, + downloads=10_000_000, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + ) + old_gen = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=12_000_000, + gguf_variants=[_gguf("Q4_K_M", 4.0)], + ) + results = rank_models( + [new_gen, old_gen], + _hw(vram_gb=24), + top_n=2, + benchmark_scores={ + "Qwen/Qwen3-8B": 56.0, # current AA/LB-derived + "Qwen/Qwen2.5-7B-Instruct": 35.0, # current source supersedes OLLB + }, + ) + assert [r.model.id for r in results][0] == "Qwen/Qwen3-8B" + + +def test_speed_estimator_differs_by_quant_and_backend(): + """Q4_K_M should run faster than F16 on the same hardware, and CUDA + should be faster than Metal at equal bandwidth.""" + from whichllm.engine.performance import estimate_tok_per_sec + from whichllm.hardware.types import GPUInfo + + model = ModelInfo( + id="t/x", + family_id="t/x", + name="x", + parameter_count=8_000_000_000, + downloads=0, + ) + q4 = GGUFVariant( + filename="x.Q4_K_M.gguf", quant_type="Q4_K_M", file_size_bytes=int(8e9 * 0.5625) + ) + f16 = GGUFVariant( + filename="x.F16.gguf", quant_type="F16", file_size_bytes=int(8e9 * 2.0) + ) + cuda = GPUInfo( + name="t-nv", + vendor="nvidia", + vram_bytes=24 * 1024**3, + memory_bandwidth_gbps=1000.0, + ) + metal = GPUInfo( + name="t-apple", + vendor="apple", + vram_bytes=24 * 1024**3, + memory_bandwidth_gbps=1000.0, + ) + q4_cuda = estimate_tok_per_sec(model, q4, cuda, "full_gpu") + f16_cuda = estimate_tok_per_sec(model, f16, cuda, "full_gpu") + q4_metal = estimate_tok_per_sec(model, q4, metal, "full_gpu") + assert q4_cuda > f16_cuda # Q4 dequant is faster than F16 matmul read + assert q4_cuda > q4_metal # CUDA beats Metal at equal bandwidth + + +def test_vram_kv_cache_scales_with_context(): + """KV-cache contribution should grow linearly with context length so + longer contexts surface as a real VRAM cost in `plan`.""" + from whichllm.engine.vram import estimate_kv_cache + + model = ModelInfo( + id="t/x", + family_id="t/x", + name="x", + parameter_count=32_000_000_000, + downloads=0, + ) + kv_4k = estimate_kv_cache(model, 4096) + kv_32k = estimate_kv_cache(model, 32768) + assert kv_32k > kv_4k * 7 # near-linear in context length + # and the absolute size at 32K should be in the gigabyte range + assert kv_32k > 2 * 1024**3 diff --git a/tests/test_quantization.py b/tests/test_quantization.py new file mode 100644 index 0000000..cc6ba7d --- /dev/null +++ b/tests/test_quantization.py @@ -0,0 +1,141 @@ +"""Tests for non-GGUF quantization inference.""" + +from whichllm.engine.quantization import ( + effective_quant_type, + estimate_weight_bytes, + infer_non_gguf_quant_type, +) +from whichllm.engine.vram import estimate_vram +from whichllm.models.types import ModelInfo + + +def _make_model(model_id: str, params: int = 14_000_000_000) -> ModelInfo: + return ModelInfo( + id=model_id, + family_id=model_id, + name=model_id.split("/")[-1], + parameter_count=params, + ) + + +def test_infer_non_gguf_awq(): + model = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ") + assert infer_non_gguf_quant_type(model.id) == "AWQ" + assert effective_quant_type(model, None) == "AWQ" + + +def test_estimate_weight_bytes_for_awq(): + model = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ", params=10_000_000_000) + assert estimate_weight_bytes(model, None) == 5_000_000_000 + + +def test_awq_vram_is_lower_than_fp16_fallback(): + awq = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ") + fp16 = _make_model("Qwen/Qwen2.5-14B-Instruct") + assert estimate_vram(awq, None, context_length=4096) < estimate_vram( + fp16, None, context_length=4096 + ) + + +def test_infer_mxfp4(): + model = _make_model("openai/gpt-oss-20b-MXFP4") + assert infer_non_gguf_quant_type(model.id) == "MXFP4" + assert effective_quant_type(model, None) == "MXFP4" + + +def test_infer_nvfp4(): + model = _make_model("nvidia/Llama-3.1-8B-Instruct-NVFP4") + assert infer_non_gguf_quant_type(model.id) == "NVFP4" + assert effective_quant_type(model, None) == "NVFP4" + + +def test_fp4_patterns_do_not_false_match_plain_ids(): + # A bare id with no fp4 token must not be mislabeled as a microscaling float. + plain = _make_model("meta-llama/Llama-3.1-8B-Instruct") + assert infer_non_gguf_quant_type(plain.id) == "FP16" + + +def test_estimate_weight_bytes_for_fp4_formats(): + mxfp4 = _make_model("openai/gpt-oss-20b-MXFP4", params=20_000_000_000) + nvfp4 = _make_model("nvidia/model-NVFP4", params=20_000_000_000) + assert estimate_weight_bytes(mxfp4, None) == int(20_000_000_000 * 0.53125) + assert estimate_weight_bytes(nvfp4, None) == int(20_000_000_000 * 0.5625) + + +def test_fp4_vram_is_lower_than_fp16_fallback(): + mxfp4 = _make_model("openai/gpt-oss-20b-MXFP4") + fp16 = _make_model("openai/gpt-oss-20b") + assert estimate_vram(mxfp4, None, context_length=4096) < estimate_vram( + fp16, None, context_length=4096 + ) + + +def test_extract_quant_type_parses_fp4_gguf_filenames(): + from whichllm.models.fetcher import _extract_quant_type + + assert _extract_quant_type("gpt-oss-20b-MXFP4.gguf") == "MXFP4" + assert _extract_quant_type("model.NVFP4.gguf") == "NVFP4" + + +def test_extract_quant_type_canonicalizes_full_precision_aliases(): + # llama.cpp publishes full-precision GGUFs as *-FP16/*-FP32; the byte and + # penalty tables key these as F16/F32, so the extractor must canonicalize. + from whichllm.models.fetcher import _extract_quant_type + + assert _extract_quant_type("Meta-Llama-3-8B-FP16.gguf") == "F16" + assert _extract_quant_type("model.FP32.gguf") == "F32" + # Canonical spellings still pass through unchanged. + assert _extract_quant_type("model-F16.gguf") == "F16" + assert _extract_quant_type("model.BF16.gguf") == "BF16" + + +def test_extract_quant_type_recognizes_ternary_gguf(): + # BitNet-class ternary GGUFs (TQ1_0/TQ2_0) are fully priced in the tables + # but were previously extracted as "unknown" and dropped at fetch. + from whichllm.models.fetcher import _extract_quant_type + + assert _extract_quant_type("BitNet-b1.58-2B-4T-TQ1_0.gguf") == "TQ1_0" + assert _extract_quant_type("model.TQ2_0.gguf") == "TQ2_0" + + +def test_estimate_gguf_size_does_not_undersize_fp16(): + # An FP16 GGUF must size at full precision (2.0 bytes/weight), not collapse + # to the Q4_K_M 0.5625 default that an unrecognized token falls back to. + from whichllm.models.fetcher import _estimate_gguf_size, _extract_quant_type + + params = 7_000_000_000 + quant = _extract_quant_type("model-FP16.gguf") + size = _estimate_gguf_size(params, quant) + assert size == params * 2 # 14 GB, not the ~3.94 GB default + assert size == _estimate_gguf_size(params, "F16") + + +def test_extract_quant_type_keys_resolve_in_byte_table(): + # Drift guard: every quant the extractor surfaces from a real GGUF filename + # must resolve in QUANT_BYTES_PER_WEIGHT, otherwise it is silently mis-sized + # by the default or dropped at fetch. Keeps the extractor and tables aligned. + from whichllm.data.quantization import QUANT_BYTES_PER_WEIGHT + from whichllm.models.fetcher import _extract_quant_type + + filenames = [ + "model-Q4_K_M.gguf", + "model-Q8_0.gguf", + "model-Q6_K.gguf", + "model-IQ4_NL.gguf", + "model-IQ3_XXS.gguf", + "model-TQ1_0.gguf", + "model-TQ2_0.gguf", + "model-F16.gguf", + "model-FP16.gguf", + "model-BF16.gguf", + "model-F32.gguf", + "model-FP32.gguf", + "model-MXFP4.gguf", + "model-NVFP4.gguf", + ] + for fname in filenames: + quant = _extract_quant_type(fname) + assert quant != "unknown", f"{fname} not recognized by extractor" + assert quant in QUANT_BYTES_PER_WEIGHT, ( + f"{fname} -> {quant!r} missing from QUANT_BYTES_PER_WEIGHT" + ) diff --git a/tests/test_r3_regressions.py b/tests/test_r3_regressions.py new file mode 100644 index 0000000..472ddf0 --- /dev/null +++ b/tests/test_r3_regressions.py @@ -0,0 +1,729 @@ +"""Regression tests for the Round 3 fixes (whichllm 0.5.1). + +Each test reproduces a specific bug class that the 0.5.1-dev ranker +exhibited and that was found by stress-testing previously unexercised +axes (Apple --gpu, family inheritance order, grouper base selection, +reasoning-model surfacing): + +- R3-1: ``--gpu "M1"`` fuzzy-matched the 1997 ATI Rage Mobility-M1 + (vendor=amd); ``--gpu "M3 Max"`` fell through to vendor=nvidia. +- R3-2: a 6.6B "imatrix-aligned" / MTP-head fork inherited its 158B + base's benchmark via family/base_model lookup. +- R3-3: a high-download downstream fork overrode the official upstream + model as the family base, corrupting ``family_id``. +- R3-4/R3-5: reasoning lines (QwQ-32B, DeepSeek-R1-Distill) had no + curated benchmark entry and never surfaced in the ranking. +""" + +from __future__ import annotations + +from whichllm.engine.ranker import rank_models +from whichllm.hardware.gpu_simulator import create_synthetic_gpu +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.benchmark import _params_compatible +from whichllm.models.benchmark_sources.aa_index import AA_INDEX_FALLBACK_2026_06_29 +from whichllm.models.benchmark_sources.livebench import LIVEBENCH_RAW_DATA +from whichllm.models.benchmark_sources.vision import VISION_FALLBACK_2026_05 +from whichllm.models.grouper import group_models +from whichllm.models.types import GGUFVariant, ModelInfo + + +def _hw( + vram_gb: int = 24, + bandwidth_gbps: float = 1000.0, + vendor: str = "nvidia", + os_name: str = "linux", + with_gpu: bool = True, +) -> HardwareInfo: + gpus = [] + if with_gpu: + gpus = [ + GPUInfo( + name="Test GPU", + vendor=vendor, + vram_bytes=vram_gb * 1024**3, + compute_capability=(8, 9) if vendor == "nvidia" else None, + memory_bandwidth_gbps=bandwidth_gbps, + ) + ] + return HardwareInfo( + gpus=gpus, + cpu_name="Test CPU", + cpu_cores=8, + has_avx2=True, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os=os_name, + ) + + +def _gguf(quant: str, size_gb: float) -> GGUFVariant: + return GGUFVariant( + filename=f"model-{quant}.gguf", + quant_type=quant, + file_size_bytes=int(size_gb * 1e9), + ) + + +# ------------------------------------------------------------------ R3-1 + + +class TestAppleSiliconSimulator: + """``--gpu`` must recognize Apple Silicon instead of fuzzy-matching + discrete-GPU database entries.""" + + def test_m1_default_is_apple_not_ati_rage_mobility(self): + gpu = create_synthetic_gpu("M1") + assert gpu.vendor == "apple", ( + f"M1 must be Apple, got vendor={gpu.vendor!r} " + f"name={gpu.name!r} (regression: fuzzy-matched ATI Rage " + "Mobility-M1)" + ) + assert "rage" not in gpu.name.lower() + # Default unified memory for the base M1 is 8 GB. + assert gpu.vram_bytes == 8 * 1024**3 + assert gpu.memory_bandwidth_gbps == 68.25 + + def test_m3_max_vram_override_apple_400gbps(self): + gpu = create_synthetic_gpu("M3 Max", vram_override_gb=64) + assert gpu.vendor == "apple", ( + f"M3 Max must be Apple, got vendor={gpu.vendor!r} " + "(regression: fell through to nvidia default)" + ) + assert gpu.vram_bytes == 64 * 1024**3 + assert gpu.memory_bandwidth_gbps == 400.0 + + def test_m2_ultra_192gb_apple_800gbps(self): + gpu = create_synthetic_gpu("M2 Ultra", vram_override_gb=192) + assert gpu.vendor == "apple" + assert gpu.vram_bytes == 192 * 1024**3 + assert gpu.memory_bandwidth_gbps == 800.0 + + def test_apple_chip_compact_form_is_recognized(self): + # Users type "M2Max" / "m2 max" / "M2 MAX" interchangeably. + for name in ("M2Max", "m2 max", "M2 MAX"): + gpu = create_synthetic_gpu(name, vram_override_gb=32) + assert gpu.vendor == "apple", f"{name!r} not recognized as Apple" + assert gpu.memory_bandwidth_gbps == 400.0 + + def test_longest_match_wins_m2_ultra_not_m2(self): + # "M2 Ultra" must not be swallowed by the "M2" entry (100 GB/s). + gpu = create_synthetic_gpu("M2 Ultra", vram_override_gb=128) + assert gpu.memory_bandwidth_gbps == 800.0 + assert gpu.memory_bandwidth_gbps != 100.0 + + +# ------------------------------------------------------------------ R3-2 + + +class TestFamilySizeInheritance: + """A small fork must not inherit a much larger base model's + benchmark score.""" + + def test_params_compatible_rejects_25x_mismatch(self): + # 6.6B vs an id that encodes 158B → ratio 0.04, must reject. + assert _params_compatible(6.6, "org/Some-Model-158B") is False + + def test_params_compatible_accepts_same_size_quant(self): + # A Q4 repack of an 8B model is still 8B → must inherit. + assert _params_compatible(7.8, "org/Llama-3-8B-GGUF") is True + + def test_params_compatible_permissive_when_no_actual_size(self): + # No actual size → cannot judge, must not block (avoids + # false-negatives that would erase legitimate inheritance). + assert _params_compatible(None, "org/Model-70B") is True + + def test_params_compatible_permissive_when_ref_has_no_size(self): + # Base id without a parseable size (e.g. DeepSeek-V4-Flash) → + # the function alone cannot guard; the ranker's + # family_dominant_params check is the backstop (next test). + assert _params_compatible(6.6, "deepseek-ai/DeepSeek-V4-Flash") is True + + def test_ranker_drops_tiny_fork_inheriting_huge_base(self): + """The real bug: jedisct1/DeepSeek-V4-Flash-imatrix-aligned + (6.6B) shared family_id with the 158B base and inherited its + leaderboard score, landing in the CPU-only top 5.""" + base = ModelInfo( + id="org/DeepSeek-Vx-Flash", + family_id="deepseek-vx-flash", + name="DeepSeek-Vx-Flash", + parameter_count=158_000_000_000, + downloads=1_000_000, + likes=1000, + gguf_variants=[], # safetensors-only official + ) + tiny_fork = ModelInfo( + id="fork/DeepSeek-Vx-Flash-mtp-aligned", + family_id="deepseek-vx-flash", + name="DeepSeek-Vx-Flash-mtp-aligned", + parameter_count=6_600_000_000, + downloads=0, + likes=0, + base_model="org/DeepSeek-Vx-Flash", + gguf_variants=[_gguf("Q8_0", 7.0)], + ) + # External leaderboard only knows the 158B base. + scores = {"org/DeepSeek-Vx-Flash": 92.0} + # Tiny VRAM: the 158B base cannot run; only the 6.6B fork fits. + hw = _hw(vram_gb=12) + ranked = rank_models( + [base, tiny_fork], + hw, + benchmark_scores=scores, + require_direct_top=False, + ) + # With the guard the fork's family/base inheritance is rejected + # entirely (status=none, score≈20). Without it the fork inherits + # the 158B base at confidence 0.6 (status=estimated, score≈59). + # The thresholds below sit firmly between those two regimes so + # the test goes red the instant the guard is removed. + tiny_res = next((r for r in ranked if r.model.id == tiny_fork.id), None) + assert tiny_res is not None, "tiny fork should still be listed" + assert tiny_res.benchmark_status == "none", ( + "tiny fork inherited a benchmark from its 158B base " + f"(status={tiny_res.benchmark_status!r}); the " + "family_dominant_params guard is not working" + ) + assert tiny_res.quality_score < 30.0, ( + "tiny fork's score reflects inherited 158B evidence " + f"(got {tiny_res.quality_score:.1f}, expected <30)" + ) + + +# ------------------------------------------------------------------ R3-3 + + +class TestGrouperUpstreamBase: + """Family base selection must follow the base_model graph, not raw + download counts.""" + + def test_official_upstream_wins_over_more_downloaded_fork(self): + official = ModelInfo( + id="Qwen/Qwen3-4B-Thinking-2507", + family_id="", + name="Qwen3-4B-Thinking-2507", + parameter_count=4_000_000_000, + downloads=494_000, + base_model=None, + ) + popular_fork = ModelInfo( + id="prefeitura-rio/Rio-3.0-Open-Mini", + family_id="", + name="Rio-3.0-Open-Mini", + parameter_count=4_000_000_000, + downloads=1_300_000, # more than the official base + base_model="Qwen/Qwen3-4B-Thinking-2507", + ) + gguf_fork = ModelInfo( + id="MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF", + family_id="", + name="Qwen3-4B-Thinking-2507-GGUF", + parameter_count=4_000_000_000, + downloads=26_000, + base_model="Qwen/Qwen3-4B-Thinking-2507", + gguf_variants=[_gguf("Q4_K_M", 2.4)], + ) + families = group_models([popular_fork, official, gguf_fork]) + # All three collapse into one family. + assert len(families) == 1 + fam = families[0] + assert fam.base_model.id == "Qwen/Qwen3-4B-Thinking-2507", ( + f"family base is {fam.base_model.id!r}; the popular fork " + "overrode the official upstream (regression R3-3)" + ) + # Every member must carry the upstream-derived family_id. + all_ids = {fam.base_model.id} | {v.id for v in fam.variants} + assert "Qwen/Qwen3-4B-Thinking-2507" in all_ids + for m in [official, popular_fork, gguf_fork]: + assert m.family_id == official.family_id + assert "rio" not in m.family_id + + def test_falls_back_to_downloads_without_upstream_reference(self): + # No member references another's base_model → keep the prior + # "most downloads, no GGUF" behaviour. + a = ModelInfo( + id="orgA/Model-7B", + family_id="", + name="Model-7B", + parameter_count=7_000_000_000, + downloads=500, + ) + b = ModelInfo( + id="orgB/Model-7B", + family_id="", + name="Model-7B", + parameter_count=7_000_000_000, + downloads=5000, + ) + families = group_models([a, b]) + assert len(families) == 1 + assert families[0].base_model.id == "orgB/Model-7B" + + +# --------------------------------------------------------------- R3-4/5 + + +class TestReasoningSurface: + """Reasoning/thinking lines must have curated benchmark anchors and + surface in the ranking.""" + + def test_qwq32b_has_curated_benchmarks(self): + assert "Qwen/QwQ-32B" in LIVEBENCH_RAW_DATA + assert "Qwen/QwQ-32B" in AA_INDEX_FALLBACK_2026_06_29 + + def test_r1_distill_family_has_curated_benchmarks(self): + for mid in ( + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + ): + assert mid in LIVEBENCH_RAW_DATA, f"{mid} missing in LB" + assert mid in AA_INDEX_FALLBACK_2026_06_29, f"{mid} missing in AA" + + def test_qwq32b_surfaces_with_curated_score(self): + qwq = ModelInfo( + id="Qwen/QwQ-32B", + family_id="qwq-32b", + name="QwQ-32B", + parameter_count=32_800_000_000, + downloads=64_000, + gguf_variants=[_gguf("Q4_K_M", 20.0)], + ) + filler = ModelInfo( + id="org/Generic-7B", + family_id="generic-7b", + name="Generic-7B", + parameter_count=7_000_000_000, + downloads=10, + gguf_variants=[_gguf("Q4_K_M", 4.5)], + ) + scores = {"Qwen/QwQ-32B": 57.0} + hw = _hw(vram_gb=48) + ranked = rank_models( + [qwq, filler], hw, benchmark_scores=scores, require_direct_top=False + ) + ids = [r.model.id for r in ranked] + assert "Qwen/QwQ-32B" in ids, "QwQ-32B did not surface in ranking" + qwq_res = next(r for r in ranked if r.model.id == "Qwen/QwQ-32B") + assert qwq_res.benchmark_status == "direct" + assert qwq_res.quality_score > 0 + + +# ----------------------------------------------------------------- R3-6 + + +class TestVisionGenerationOrder: + """``--profile vision`` had no VLM benchmark source, so only a + two-generations-old Qwen2-VL-7B had a direct hit and outranked the + current Qwen3-VL-32B even on an 80 GB GPU.""" + + def test_curated_vision_scores_respect_generation(self): + v = VISION_FALLBACK_2026_05 + # The whole point: newest generation must outscore the oldest. + assert ( + v["Qwen/Qwen3-VL-32B-Instruct"] + > v["Qwen/Qwen2.5-VL-32B-Instruct"] + > v["Qwen/Qwen2-VL-7B-Instruct"] + ) + # Two-generations-old 7B must sit in the low band so it cannot + # win a vision ranking by virtue of being the only direct hit. + assert v["Qwen/Qwen2-VL-7B-Instruct"] <= 35.0 + + def test_qwen3_vl_outranks_legacy_qwen2_vl_on_vision_profile(self): + new_vlm = ModelInfo( + id="Qwen/Qwen3-VL-32B-Instruct", + family_id="qwen3-vl-32b", + name="Qwen3-VL-32B-Instruct", + parameter_count=33_400_000_000, + downloads=1_500_000, + gguf_variants=[_gguf("Q4_K_M", 20.0)], + ) + legacy_vlm = ModelInfo( + id="Qwen/Qwen2-VL-7B-Instruct", + family_id="qwen2-vl-7b", + name="Qwen2-VL-7B-Instruct", + parameter_count=8_300_000_000, + downloads=2_000_000, # more downloads — must NOT win + gguf_variants=[_gguf("Q4_K_M", 5.0)], + ) + # Feed the curated vision scores exactly as the merge would. + scores = { + "Qwen/Qwen3-VL-32B-Instruct": VISION_FALLBACK_2026_05[ + "Qwen/Qwen3-VL-32B-Instruct" + ], + "Qwen/Qwen2-VL-7B-Instruct": VISION_FALLBACK_2026_05[ + "Qwen/Qwen2-VL-7B-Instruct" + ], + } + hw = _hw(vram_gb=80) + ranked = rank_models( + [legacy_vlm, new_vlm], + hw, + benchmark_scores=scores, + task_profile="vision", + require_direct_top=False, + ) + assert ranked, "no vision models ranked" + assert ranked[0].model.id == "Qwen/Qwen3-VL-32B-Instruct", ( + "legacy Qwen2-VL-7B outranked the current Qwen3-VL-32B on " + f"the vision profile (got {ranked[0].model.id})" + ) + + +# ----------------------------------------------------------------- R3-7 + + +class TestApplePartialOffloadPenalty: + """Partial-offload penalty must respect memory architecture: Apple + unified memory has no PCIe cliff, so the discrete 0.45x penalty made + DeepSeek-R1-class models on M2/M3 Ultra report ~1.7 t/s vs a + real-world 4-15.""" + + def _model_variant(self): + m = ModelInfo( + id="deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + family_id="deepseek-r1-distill-qwen-32b", + name="DeepSeek-R1-Distill-Qwen-32B", + parameter_count=32_800_000_000, + ) + v = _gguf("Q4_K_M", 20.0) + return m, v + + def test_apple_partial_offload_keeps_most_of_full_speed(self): + from whichllm.engine.performance import estimate_tok_per_sec + + m, v = self._model_variant() + apple = GPUInfo( + name="M2 Ultra", + vendor="apple", + vram_bytes=192 * 1024**3, + memory_bandwidth_gbps=800.0, + ) + full = estimate_tok_per_sec(m, v, apple, "full_gpu") + partial = estimate_tok_per_sec(m, v, apple, "partial_offload") + assert full > 0 + ratio = partial / full + # 0.85 with the fix; 0.45 if the vendor branch is removed. The + # 0.7 threshold sits between the two regimes. + assert ratio > 0.7, ( + f"Apple partial-offload ratio {ratio:.2f} — the discrete " + "0.45x PCIe penalty is being wrongly applied to unified memory" + ) + + def test_discrete_partial_offload_still_takes_pcie_penalty(self): + from whichllm.engine.performance import estimate_tok_per_sec + + m, v = self._model_variant() + nvidia = GPUInfo( + name="RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ) + full = estimate_tok_per_sec(m, v, nvidia, "full_gpu") + partial = estimate_tok_per_sec(m, v, nvidia, "partial_offload") + assert full > 0 + ratio = partial / full + # Discrete GPUs must keep the steep PCIe penalty (~0.45). + assert ratio < 0.6, ( + f"discrete partial-offload ratio {ratio:.2f} — the PCIe " + "penalty is too weak; offload should hurt on NVIDIA/AMD" + ) + + def test_apple_partial_faster_than_discrete_partial_same_bandwidth(self): + from whichllm.engine.performance import estimate_tok_per_sec + + m, v = self._model_variant() + # Same nominal bandwidth so the only difference is the + # architecture-aware offload penalty. + apple = GPUInfo( + name="Apple", + vendor="apple", + vram_bytes=192 * 1024**3, + memory_bandwidth_gbps=900.0, + ) + nvidia = GPUInfo( + name="NVIDIA", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=900.0, + ) + a = estimate_tok_per_sec(m, v, apple, "partial_offload") + n = estimate_tok_per_sec(m, v, nvidia, "partial_offload") + assert a > n, ( + "Apple unified-memory partial offload should beat discrete " + f"PCIe-bound partial offload at equal bandwidth ({a:.1f} vs " + f"{n:.1f})" + ) + + +# ----------------------------------------------------------------- R3-8 + + +class TestMoESpeedEstimation: + """MoE speed estimation should use active params without letting + high-bandwidth GPUs turn sparse models into unrealistic outliers.""" + + def test_qwen3_next_strix_halo_matches_reported_generation_speed(self): + from whichllm.engine.performance import estimate_tok_per_sec + + model = ModelInfo( + id="Qwen/Qwen3-Next-80B-A3B-Instruct", + family_id="qwen3-next-80b-a3b", + name="Qwen3-Next-80B-A3B-Instruct", + parameter_count=79_670_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + variant = GGUFVariant( + filename="qwen3next-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=int(45.17 * 1024**3), + ) + strix_halo = GPUInfo( + name="STRXLGEN", + vendor="amd", + vram_bytes=0, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + + speed = estimate_tok_per_sec(model, variant, strix_halo, "full_gpu") + + assert 40.0 <= speed <= 50.0 + + def test_unknown_ultra_sparse_moe_uses_active_params_on_strix_halo(self): + from whichllm.engine.performance import estimate_tok_per_sec + + model = ModelInfo( + id="unknown/Experimental-80B-A3B", + family_id="experimental-80b-a3b", + name="Experimental-80B-A3B", + parameter_count=79_670_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + variant = GGUFVariant( + filename="experimental-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=int(45.17 * 1024**3), + ) + strix_halo = GPUInfo( + name="STRXLGEN", + vendor="amd", + vram_bytes=0, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + + speed = estimate_tok_per_sec(model, variant, strix_halo, "full_gpu") + + assert 40.0 <= speed <= 50.0 + + def test_qwen3_30b_a3b_strix_halo_no_longer_uses_legacy_floor(self): + from whichllm.engine.performance import estimate_tok_per_sec + + model = ModelInfo( + id="Qwen/Qwen3-30B-A3B", + family_id="qwen3-30b-a3b", + name="Qwen3-30B-A3B", + parameter_count=30_000_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + variant = GGUFVariant( + filename="qwen3-30b-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=int(17.1 * 1024**3), + ) + strix_halo = GPUInfo( + name="STRXLGEN", + vendor="amd", + vram_bytes=0, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + + speed = estimate_tok_per_sec(model, variant, strix_halo, "full_gpu") + + assert 50.0 <= speed <= 70.0 + + def test_high_bandwidth_gpu_keeps_moe_kernel_floor(self): + from whichllm.engine.performance import estimate_tok_per_sec + + model = ModelInfo( + id="Qwen/Qwen3-30B-A3B", + family_id="qwen3-30b-a3b", + name="Qwen3-30B-A3B", + parameter_count=30_000_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + variant = _gguf("Q5_K_M", 20.6) + rtx_4090 = GPUInfo( + name="RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ) + + speed = estimate_tok_per_sec(model, variant, rtx_4090, "full_gpu") + + assert 100.0 <= speed <= 150.0 + + def test_deepseek_v4_flash_synthetic_q4_does_not_fit_strix_halo_96gb(self): + deepseek = ModelInfo( + id="deepseek-ai/DeepSeek-V4-Flash", + family_id="deepseek-v4-flash", + name="DeepSeek-V4-Flash", + parameter_count=284_000_000_000, + parameter_count_active=13_000_000_000, + is_moe=True, + downloads=1_000_000, + gguf_variants=[], + ) + qwen_dense = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3.6-27b", + name="Qwen3.6-27B", + parameter_count=27_000_000_000, + downloads=100_000, + gguf_variants=[], + ) + hardware = HardwareInfo( + gpus=[ + GPUInfo( + name="Strix Halo", + vendor="amd", + vram_bytes=96 * 1024**3, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + ], + cpu_name="Ryzen AI MAX+ 395", + cpu_cores=16, + ram_bytes=128 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + + results = rank_models( + [deepseek, qwen_dense], + hardware, + top_n=5, + quant_filter="Q4_K_M", + benchmark_scores={ + "deepseek-ai/DeepSeek-V4-Flash": 87.0, + "Qwen/Qwen3.6-27B": 84.0, + }, + ) + + ids = [r.model.id for r in results] + assert "deepseek-ai/DeepSeek-V4-Flash" not in ids + assert "Qwen/Qwen3.6-27B" in ids + + +class TestSpeedUncertainty: + def test_strix_halo_moe_speed_is_medium_confidence_with_range(self): + from whichllm.engine.performance import estimate_speed_uncertainty + + model = ModelInfo( + id="unknown/Experimental-80B-A3B", + family_id="experimental-80b-a3b", + name="Experimental-80B-A3B", + parameter_count=79_670_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + variant = GGUFVariant( + filename="experimental-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=int(45.17 * 1024**3), + ) + strix_halo = GPUInfo( + name="Strix Halo", + vendor="amd", + vram_bytes=96 * 1024**3, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + + confidence, speed_range, notes = estimate_speed_uncertainty( + model, variant, strix_halo, "full_gpu", 48.0 + ) + + assert confidence == "medium" + assert speed_range == (28.8, 76.8) + assert any("shared-memory APU" in note for note in notes) + + def test_apple_silicon_moe_speed_is_low_confidence(self): + from whichllm.engine.performance import estimate_speed_uncertainty + + model = ModelInfo( + id="google/gemma-4-26B-A4B-it", + family_id="gemma-4-26b-a4b-it", + name="gemma-4-26B-A4B-it", + parameter_count=26_000_000_000, + parameter_count_active=3_800_000_000, + is_moe=True, + ) + variant = _gguf("Q4_K_M", 15.0) + apple = GPUInfo( + name="M3 Max", + vendor="apple", + vram_bytes=96 * 1024**3, + memory_bandwidth_gbps=400.0, + shared_memory=True, + ) + + confidence, speed_range, notes = estimate_speed_uncertainty( + model, variant, apple, "full_gpu", 30.0 + ) + + assert confidence == "low" + assert speed_range == (10.5, 60.0) + assert any("Metal/MLX" in note for note in notes) + + def test_synthetic_gguf_rank_result_exposes_speed_uncertainty(self): + model = ModelInfo( + id="Qwen/Qwen3-30B-A3B", + family_id="qwen3-30b-a3b", + name="Qwen3-30B-A3B", + parameter_count=30_000_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + downloads=1_000_000, + ) + hardware = HardwareInfo( + gpus=[ + GPUInfo( + name="Strix Halo", + vendor="amd", + vram_bytes=96 * 1024**3, + memory_bandwidth_gbps=256.0, + shared_memory=True, + ) + ], + cpu_name="Ryzen AI MAX+ 395", + cpu_cores=16, + ram_bytes=128 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + + result = rank_models( + [model], + hardware, + top_n=1, + quant_filter="Q4_K_M", + benchmark_scores={"Qwen/Qwen3-30B-A3B": 80.0}, + )[0] + + assert result.speed_confidence == "medium" + assert result.speed_range_tok_per_sec is not None + assert result.speed_range_tok_per_sec[0] < result.estimated_tok_per_sec + assert result.speed_range_tok_per_sec[1] > result.estimated_tok_per_sec + assert any("synthetic GGUF" in note for note in result.speed_notes) diff --git a/tests/test_ranker.py b/tests/test_ranker.py new file mode 100644 index 0000000..3480cef --- /dev/null +++ b/tests/test_ranker.py @@ -0,0 +1,1155 @@ +"""Tests for ranking behavior.""" + +from whichllm.engine.quantization import effective_quant_type +from whichllm.engine.ranker import _partial_offload_quality_factor, rank_models +from whichllm.hardware.types import GPUInfo, HardwareInfo +from whichllm.models.types import GGUFVariant, ModelInfo + + +def _make_hardware( + vram_gb: int = 24, + bandwidth_gbps: float = 80.0, + vendor: str = "nvidia", + os_name: str = "linux", + with_gpu: bool = True, +) -> HardwareInfo: + gpus = [] + if with_gpu: + gpus = [ + GPUInfo( + name="Test GPU", + vendor=vendor, + vram_bytes=vram_gb * 1024**3, + compute_capability=(8, 9) if vendor == "nvidia" else None, + memory_bandwidth_gbps=bandwidth_gbps, + ), + ] + return HardwareInfo( + gpus=gpus, + cpu_name="Test CPU", + cpu_cores=8, + has_avx2=True, + ram_bytes=64 * 1024**3, + disk_free_bytes=500 * 1024**3, + os=os_name, + ) + + +def test_ranker_picks_highest_scoring_variant(): + # On a fast GPU (≥800 GB/s) both quants run well above the comfort + # threshold, so the F16 quality bonus dominates and F16 wins. On a slow + # GPU the speed gap flips the choice — that's exercised separately. + model = ModelInfo( + id="org/Test-8B-GGUF", + family_id="org/Test-8B-GGUF", + name="Test-8B-GGUF", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="test-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + GGUFVariant( + filename="test-F16.gguf", + quant_type="F16", + file_size_bytes=5_000_000_000, + ), + ], + ) + hw = _make_hardware(bandwidth_gbps=900.0) + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={"org/Test-8B-GGUF": 70.0}, + ) + assert results + assert results[0].gguf_variant is not None + assert results[0].gguf_variant.quant_type == "F16" + + +def test_quant_filter_applies_to_non_gguf_models(): + model = ModelInfo( + id="Qwen/Qwen2.5-14B-Instruct-AWQ", + family_id="qwen2.5-14b", + name="Qwen2.5-14B-Instruct-AWQ", + parameter_count=14_000_000_000, + downloads=1000, + likes=100, + ) + hw = _make_hardware(vram_gb=24, bandwidth_gbps=300.0) + + awq_only = rank_models([model], hw, top_n=5, quant_filter="AWQ") + q4_only = rank_models([model], hw, top_n=5, quant_filter="Q4_K_M") + + assert len(awq_only) == 1 + assert q4_only == [] + + +def test_quant_filter_matches_mxfp4_non_gguf_model(): + model = ModelInfo( + id="openai/gpt-oss-20b-MXFP4", + family_id="gpt-oss-20b", + name="gpt-oss-20b-MXFP4", + parameter_count=20_000_000_000, + downloads=1000, + likes=100, + ) + # Linux + NVIDIA: non-GGUF formats are runnable, so the filter resolves. + hw = _make_hardware(vram_gb=24, bandwidth_gbps=900.0) + + mxfp4_only = rank_models([model], hw, top_n=5, quant_filter="MXFP4") + nvfp4_only = rank_models([model], hw, top_n=5, quant_filter="NVFP4") + + assert len(mxfp4_only) == 1 + # The label surfaced in the output table (display.py uses the same call). + assert ( + effective_quant_type(mxfp4_only[0].model, mxfp4_only[0].gguf_variant) == "MXFP4" + ) + assert nvfp4_only == [] + + +def test_darwin_backend_filters_out_fp4_non_gguf_models(): + mxfp4_model = ModelInfo( + id="openai/gpt-oss-20b-MXFP4", + family_id="gpt-oss-20b", + name="gpt-oss-20b-MXFP4", + parameter_count=20_000_000_000, + downloads=1000, + likes=100, + ) + hw = _make_hardware( + vram_gb=64, bandwidth_gbps=400.0, vendor="apple", os_name="darwin" + ) + results = rank_models([mxfp4_model], hw, top_n=10) + assert results == [] + + +def test_darwin_backend_filters_out_non_gguf_models(): + awq_model = ModelInfo( + id="Qwen/Qwen3-8B-AWQ", + family_id="qwen3-8b-awq", + name="Qwen3-8B-AWQ", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + ) + gguf_model = ModelInfo( + id="Qwen/Qwen3-8B-GGUF", + family_id="qwen3-8b-gguf", + name="Qwen3-8B-GGUF", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="a-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware( + vram_gb=16, bandwidth_gbps=200.0, vendor="apple", os_name="darwin" + ) + results = rank_models([awq_model, gguf_model], hw, top_n=10) + assert len(results) == 1 + assert results[0].model.id == "Qwen/Qwen3-8B-GGUF" + + +def test_cpu_only_backend_filters_out_non_gguf_models(): + awq_model = ModelInfo( + id="Qwen/Qwen3-8B-AWQ", + family_id="qwen3-8b-awq", + name="Qwen3-8B-AWQ", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + ) + gguf_model = ModelInfo( + id="Qwen/Qwen3-8B-GGUF", + family_id="qwen3-8b-gguf", + name="Qwen3-8B-GGUF", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="a-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware(with_gpu=False, os_name="linux") + results = rank_models([awq_model, gguf_model], hw, top_n=10) + assert len(results) == 1 + assert results[0].model.id == "Qwen/Qwen3-8B-GGUF" + + +def _gguf_model(model_id: str, family_id: str, downloads: int) -> ModelInfo: + return ModelInfo( + id=model_id, + family_id=family_id, + name=model_id.split("/")[-1], + parameter_count=7_000_000_000, + downloads=downloads, + likes=downloads // 10, + gguf_variants=[ + GGUFVariant( + filename=f"{family_id}-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + ], + ) + + +def test_rank_models_clamps_non_positive_top_n(): + # Several distinct families so the full ranking has multiple entries; only + # then is the slice hazard observable. + hw = _make_hardware(vram_gb=24, bandwidth_gbps=300.0) + models = [ + _gguf_model("org/Alpha-7B-GGUF", "alpha-7b", 1000), + _gguf_model("org/Beta-7B-GGUF", "beta-7b", 900), + _gguf_model("org/Gamma-7B-GGUF", "gamma-7b", 800), + ] + + full = rank_models(models, hw, top_n=10) + assert len(full) >= 2 # guard is only meaningful with several results + + # 0 and negative requests must yield nothing, never a slice-from-the-end + # subset: ``results[:-1]`` would otherwise return all-but-last. + assert rank_models(models, hw, top_n=0) == [] + assert rank_models(models, hw, top_n=-1) == [] + # Positive requests are unaffected. + assert len(rank_models(models, hw, top_n=2)) == 2 + + +def test_popularity_has_no_effect_with_direct_benchmark(): + model_low_pop = ModelInfo( + id="Qwen/test-8b-lowpop", + family_id="qwen-test-8b-lowpop", + name="test-8b-lowpop", + parameter_count=8_000_000_000, + downloads=100, + likes=5, + gguf_variants=[ + GGUFVariant( + filename="test-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + ], + ) + model_high_pop = ModelInfo( + id="Qwen/test-8b-highpop", + family_id="qwen-test-8b-highpop", + name="test-8b-highpop", + parameter_count=8_000_000_000, + downloads=1_000_000, + likes=10_000, + gguf_variants=[ + GGUFVariant( + filename="test-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [model_low_pop, model_high_pop], + hw, + top_n=2, + benchmark_scores={ + "Qwen/test-8b-lowpop": 70.0, + "Qwen/test-8b-highpop": 70.0, + }, + ) + assert len(results) == 2 + assert abs(results[0].quality_score - results[1].quality_score) < 1e-9 + + +def test_general_profile_excludes_specialized_models(): + general_model = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="a-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + coding_model = ModelInfo( + id="Qwen/Qwen2.5-Coder-7B-Instruct", + family_id="qwen2.5-coder-7b", + name="Qwen2.5-Coder-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [general_model, coding_model], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-7B-Instruct": 70.0, + "Qwen/Qwen2.5-Coder-7B-Instruct": 75.0, + }, + task_profile="general", + ) + assert len(results) == 1 + assert "Coder" not in results[0].model.id + + +def test_require_direct_top_prioritizes_direct_benchmark(): + direct_model = ModelInfo( + id="Qwen/direct-7b", + family_id="qwen-direct-7b", + name="direct-7b", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="d-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + estimated_model = ModelInfo( + id="Qwen/Qwen3-9B", + family_id="qwen3-9b", + name="Qwen3-9B", + parameter_count=9_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="e-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=5_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [direct_model, estimated_model], + hw, + top_n=10, + benchmark_scores={ + "Qwen/direct-7b": 65.0, + # Qwen3のラインスコアだけ与えてestimatedを作る + "Qwen/Qwen3-32B": 80.0, + }, + task_profile="any", + require_direct_top=True, + ) + assert len(results) == 2 + assert results[0].benchmark_status == "direct" + + +def test_min_params_filter_excludes_small_models(): + small = ModelInfo( + id="Qwen/Qwen2.5-3B-Instruct", + family_id="qwen2.5-3b", + name="Qwen2.5-3B-Instruct", + parameter_count=3_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="s-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=1_700_000_000, + ), + ], + ) + large = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="l-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [small, large], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-3B-Instruct": 90.0, + "Qwen/Qwen2.5-7B-Instruct": 70.0, + }, + task_profile="any", + min_params_b=7.0, + ) + assert len(results) == 1 + assert results[0].model.id == "Qwen/Qwen2.5-7B-Instruct" + + +def test_general_profile_prefers_full_gpu_when_direct_is_partial(): + # Direct-evidence model that won't fit on 8GB even after Q4_K_M synthesis + # (72B * 0.56 ≈ 40GB → partial_offload). + partial_direct = ModelInfo( + id="Qwen/Qwen2.5-72B-Instruct", + family_id="qwen2.5-72b", + name="Qwen2.5-72B-Instruct", + parameter_count=72_000_000_000, + downloads=1000, + likes=100, + ) + full_gpu_estimated = ModelInfo( + id="Qwen/Qwen3-9B-AWQ", + family_id="qwen3-9b", + name="Qwen3-9B-AWQ", + parameter_count=9_000_000_000, + downloads=1000, + likes=100, + ) + hw = _make_hardware(vram_gb=8, bandwidth_gbps=272.0) + results = rank_models( + [partial_direct, full_gpu_estimated], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-72B-Instruct": 80.0, # direct + "Qwen/Qwen3-32B": 85.0, # line inherited for Qwen3-9B + }, + task_profile="general", + require_direct_top=True, + ) + assert results + assert results[0].fit_type == "full_gpu" + assert results[0].model.id == "Qwen/Qwen3-9B-AWQ" + + +def test_family_dedup_prefers_direct_when_enabled(): + # 同一family内でfit条件が同等なら、directを優先する + direct_base = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + ) + estimated_variant = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct-GGUF", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct-GGUF", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="x-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware(vram_gb=16, bandwidth_gbps=272.0) + results = rank_models( + [direct_base, estimated_variant], + hw, + top_n=10, + benchmark_scores={"Qwen/Qwen2.5-7B-Instruct": 75.0}, + task_profile="general", + require_direct_top=True, + min_params_b=7.0, + ) + assert len(results) == 1 + assert results[0].model.id == "Qwen/Qwen2.5-7B-Instruct" + assert results[0].benchmark_status == "direct" + + +def test_full_gpu_estimated_ranks_above_partial_direct(): + # Use 72B model so Q4_K_M synthesis still doesn't fit 8GB — preserves the + # "direct evidence, but model is too big" half of this scenario. + partial_direct = ModelInfo( + id="Qwen/Qwen2.5-72B-Instruct", + family_id="qwen2.5-72b", + name="Qwen2.5-72B-Instruct", + parameter_count=72_000_000_000, + downloads=1000, + likes=100, + ) + full_gpu_estimated = ModelInfo( + id="Qwen/Qwen3-8B-AWQ", + family_id="qwen3-8b", + name="Qwen3-8B-AWQ", + parameter_count=8_000_000_000, + downloads=1000, + likes=100, + ) + hw = _make_hardware(vram_gb=8, bandwidth_gbps=272.0) + results = rank_models( + [partial_direct, full_gpu_estimated], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-72B-Instruct": 75.0, # direct but partial + "Qwen/Qwen3-32B": 85.0, # estimated but full gpu + }, + task_profile="general", + require_direct_top=True, + min_params_b=7.0, + ) + # The full-GPU candidate must always win over a partial-offload one of + # comparable quality. The partial 72B may or may not be retained + # depending on whether its sub-2 t/s estimate trips the speed floor — + # either way the full-GPU 8B should be #1. + assert results + assert results[0].fit_type == "full_gpu" + assert results[0].model.id == "Qwen/Qwen3-8B-AWQ" + + +def test_strong_partial_offload_not_buried_below_weaker_full_gpu(): + strong_partial = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3.6-27b", + name="Qwen3.6-27B", + parameter_count=27_800_000_000, + downloads=5_300_000, + likes=10_000, + gguf_variants=[ + GGUFVariant( + filename="qwen3.6-27b-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=15 * 1024**3, + ) + ], + ) + full_gpu_14b = ModelInfo( + id="Qwen/Qwen3-14B", + family_id="qwen3-14b", + name="Qwen3-14B", + parameter_count=14_800_000_000, + downloads=1_600_000, + likes=5_000, + gguf_variants=[ + GGUFVariant( + filename="qwen3-14b-q5_k_m.gguf", + quant_type="Q5_K_M", + file_size_bytes=9 * 1024**3, + ) + ], + ) + full_gpu_8b = ModelInfo( + id="Qwen/Qwen3-8B", + family_id="qwen3-8b", + name="Qwen3-8B", + parameter_count=8_200_000_000, + downloads=11_000_000, + likes=5_000, + gguf_variants=[ + GGUFVariant( + filename="qwen3-8b-q5_k_m.gguf", + quant_type="Q5_K_M", + file_size_bytes=5 * 1024**3, + ) + ], + ) + old_full_gpu = ModelInfo( + id="google/gemma-2-9b-it", + family_id="gemma-2-9b-it", + name="gemma-2-9b-it", + parameter_count=9_200_000_000, + downloads=400_000, + likes=1_000, + gguf_variants=[ + GGUFVariant( + filename="gemma-2-9b-q5_k_m.gguf", + quant_type="Q5_K_M", + file_size_bytes=5_500_000_000, + ) + ], + ) + hardware = HardwareInfo( + gpus=[ + GPUInfo( + name="RTX 3060", + vendor="nvidia", + vram_bytes=12 * 1024**3, + compute_capability=(8, 6), + memory_bandwidth_gbps=360.0, + ) + ], + cpu_name="Test CPU", + cpu_cores=6, + has_avx2=True, + ram_bytes=32 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="windows", + ) + + results = rank_models( + [strong_partial, full_gpu_14b, full_gpu_8b, old_full_gpu], + hardware, + top_n=10, + benchmark_scores={ + "Qwen/Qwen3.6-27B": 83.5, + "Qwen/Qwen3-14B": 66.7, + "Qwen/Qwen3-8B": 56.1, + "google/gemma-2-9b-it": 35.1, + }, + task_profile="any", + ) + + ids = [r.model.id for r in results] + assert ids.index("Qwen/Qwen3.6-27B") < ids.index("Qwen/Qwen3-8B") + assert ids.index("Qwen/Qwen3.6-27B") < ids.index("google/gemma-2-9b-it") + strong = next(r for r in results if r.model.id == "Qwen/Qwen3.6-27B") + assert strong.fit_type == "partial_offload" + assert ( + strong.quality_score + > next(r for r in results if r.model.id == "Qwen/Qwen3-8B").quality_score + ) + + +def test_moe_partial_offload_penalty_uses_active_working_set(): + dense = ModelInfo( + id="example/Dense-30B", + family_id="dense-30b", + name="Dense-30B", + parameter_count=30_000_000_000, + ) + moe = ModelInfo( + id="example/MoE-30B-A3B", + family_id="moe-30b-a3b", + name="MoE-30B-A3B", + parameter_count=30_000_000_000, + parameter_count_active=3_000_000_000, + is_moe=True, + ) + + assert _partial_offload_quality_factor(dense, 0.80) == 0.42 + assert _partial_offload_quality_factor(moe, 0.80) >= 0.66 + + +def test_evidence_strict_filters_out_estimated_models(): + direct_model = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="d-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + estimated_model = ModelInfo( + id="Qwen/Qwen3-14B-Instruct-GGUF", + family_id="qwen3-14b", + name="Qwen3-14B-Instruct-GGUF", + parameter_count=14_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="e-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=8_000_000_000, + ), + ], + ) + hw = _make_hardware(vram_gb=24, bandwidth_gbps=300.0) + results = rank_models( + [direct_model, estimated_model], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-7B-Instruct": 70.0, + "Qwen/Qwen3-32B-Instruct": 85.0, # Qwen3-14B には line 推定が入る + }, + task_profile="any", + evidence_filter="strict", + ) + assert len(results) == 1 + assert results[0].model.id == "Qwen/Qwen2.5-7B-Instruct" + assert results[0].benchmark_status == "direct" + + +def test_evidence_base_keeps_base_model_match_and_drops_line_interp(): + direct_model = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + ) + base_match_model = ModelInfo( + id="ISTA-DASLab/gemma-3-27b-it-GPTQ-4b-128g", + family_id="gemma-3-27b", + name="gemma-3-27b-it-GPTQ-4b-128g", + parameter_count=27_000_000_000, + downloads=1000, + likes=100, + base_model="google/gemma-3-27b-it", + ) + line_interp_model = ModelInfo( + id="Qwen/Qwen3-14B-Instruct-GGUF", + family_id="qwen3-14b", + name="Qwen3-14B-Instruct-GGUF", + parameter_count=14_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="f-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=8_000_000_000, + ), + ], + ) + hw = _make_hardware(vram_gb=24, bandwidth_gbps=300.0) + results = rank_models( + [direct_model, base_match_model, line_interp_model], + hw, + top_n=10, + benchmark_scores={ + "Qwen/Qwen2.5-7B-Instruct": 70.0, + "google/gemma-3-27b-it": 82.0, + "Qwen/Qwen3-32B-Instruct": 85.0, + }, + task_profile="any", + evidence_filter="base", + ) + ids = {r.model.id for r in results} + assert "Qwen/Qwen2.5-7B-Instruct" in ids + assert "ISTA-DASLab/gemma-3-27b-it-GPTQ-4b-128g" in ids + assert "Qwen/Qwen3-14B-Instruct-GGUF" not in ids + + +def test_unknown_speed_heavy_partial_offload_does_not_top_rank(): + heavy_partial = ModelInfo( + id="Qwen/Qwen3.6-27B", + family_id="qwen3.6-27b", + name="Qwen3.6-27B", + parameter_count=27_800_000_000, + downloads=1_000_000, + likes=10_000, + gguf_variants=[ + GGUFVariant( + filename="qwen3.6-27b-q8_0.gguf", + quant_type="Q8_0", + file_size_bytes=29_500_000_000, + ) + ], + ) + full_gpu = ModelInfo( + id="Qwen/Qwen3-8B", + family_id="qwen3-8b", + name="Qwen3-8B", + parameter_count=8_000_000_000, + downloads=500_000, + likes=5_000, + gguf_variants=[ + GGUFVariant( + filename="qwen3-8b-q4_k_m.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ) + ], + ) + hardware = HardwareInfo( + gpus=[ + GPUInfo( + name="Unknown 6GB NVIDIA GPU", + vendor="nvidia", + vram_bytes=6 * 1024**3, + compute_capability=(8, 6), + memory_bandwidth_gbps=None, + ) + ], + cpu_name="Test CPU", + cpu_cores=8, + has_avx2=True, + ram_bytes=32 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="windows", + ) + + results = rank_models( + [heavy_partial, full_gpu], + hardware, + top_n=2, + benchmark_scores={ + "Qwen/Qwen3.6-27B": 84.0, + "Qwen/Qwen3-8B": 62.0, + }, + ) + + assert results + assert results[0].model.id == "Qwen/Qwen3-8B" + assert results[0].fit_type == "full_gpu" + heavy = next((r for r in results if r.model.id == "Qwen/Qwen3.6-27B"), None) + if heavy is not None: + assert heavy.fit_type == "partial_offload" + assert heavy.offload_ratio >= 0.70 + assert heavy.estimated_tok_per_sec == 0.0 + + +def test_fit_filter_full_gpu_excludes_partial_offload_and_cpu_only(): + partial = ModelInfo( + id="org/Test-30B-GGUF", + family_id="test-30b", + name="Test-30B-GGUF", + parameter_count=30_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="test-30b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=18_000_000_000, + ) + ], + ) + full = ModelInfo( + id="org/Test-7B-GGUF", + family_id="test-7b", + name="Test-7B-GGUF", + parameter_count=7_000_000_000, + downloads=900, + likes=90, + gguf_variants=[ + GGUFVariant( + filename="test-7b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ) + ], + ) + cpu_only = ModelInfo( + id="org/Test-60B-GGUF", + family_id="test-60b", + name="Test-60B-GGUF", + parameter_count=60_000_000_000, + downloads=800, + likes=80, + gguf_variants=[ + GGUFVariant( + filename="test-60b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=36_000_000_000, + ) + ], + ) + hw = _make_hardware(vram_gb=8, bandwidth_gbps=300.0) + results = rank_models( + [partial, full, cpu_only], + hw, + top_n=10, + fit_filter="full_gpu", + task_profile="any", + require_direct_top=False, + ) + + assert [r.model.id for r in results] == ["org/Test-7B-GGUF"] + assert results[0].fit_type == "full_gpu" + + +def test_fit_filter_full_gpu_returns_empty_when_no_full_gpu_candidate(): + partial = ModelInfo( + id="org/Test-30B-GGUF", + family_id="test-30b", + name="Test-30B-GGUF", + parameter_count=30_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="test-30b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=18_000_000_000, + ) + ], + ) + hw = _make_hardware(vram_gb=8, bandwidth_gbps=300.0) + results = rank_models( + [partial], + hw, + top_n=10, + fit_filter="full_gpu", + task_profile="any", + require_direct_top=False, + ) + + assert results == [] + + +def test_multi_gpu_speed_confidence_is_low(): + from whichllm.engine.performance import estimate_tok_per_sec + + model = ModelInfo( + id="org/Test-34B-GGUF", + family_id="org/Test-34B-GGUF", + name="Test-34B-GGUF", + parameter_count=34_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="test-34b-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=22 * 1024**3, + ) + ], + ) + hw = HardwareInfo( + gpus=[ + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ), + GPUInfo( + name="NVIDIA GeForce RTX 4090", + vendor="nvidia", + vram_bytes=24 * 1024**3, + compute_capability=(8, 9), + memory_bandwidth_gbps=1008.0, + ), + ], + cpu_name="Test CPU", + cpu_cores=16, + has_avx2=True, + ram_bytes=128 * 1024**3, + disk_free_bytes=500 * 1024**3, + os="linux", + ) + + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={"org/Test-34B-GGUF": 70.0}, + ) + + assert results + assert results[0].fit_type == "full_gpu" + assert results[0].uses_multi_gpu is True + assert results[0].speed_confidence == "low" + single_gpu_speed = estimate_tok_per_sec( + model, + model.gguf_variants[0], + hw.gpus[0], + "full_gpu", + ) + assert results[0].estimated_tok_per_sec == single_gpu_speed * 0.70 + assert any("Multi-GPU speed depends" in note for note in results[0].speed_notes) + + +def test_benchmark_source_and_confidence_exposed_for_direct(): + model = ModelInfo( + id="Qwen/Qwen2.5-7B-Instruct", + family_id="qwen2.5-7b", + name="Qwen2.5-7B-Instruct", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="a-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={"Qwen/Qwen2.5-7B-Instruct": 75.0}, + task_profile="any", + ) + assert results + assert results[0].benchmark_status == "direct" + assert results[0].benchmark_source == "direct" + assert results[0].benchmark_confidence == 1.0 + + +def test_benchmark_source_and_confidence_exposed_for_estimated(): + model = ModelInfo( + id="Qwen/Qwen3-14B-Instruct-GGUF", + family_id="qwen3-14b", + name="Qwen3-14B-Instruct-GGUF", + parameter_count=14_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="e-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=8_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={"Qwen/Qwen3-32B-Instruct": 85.0}, + task_profile="any", + ) + assert results + assert results[0].benchmark_status == "estimated" + assert results[0].benchmark_source == "line_interp" + assert 0.0 < results[0].benchmark_confidence < 1.0 + + +def test_benchmark_source_and_confidence_exposed_for_self_reported(): + model = ModelInfo( + id="someorg/mystery-7B", + family_id="mystery-7b", + name="mystery-7B", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + benchmark_scores={"hf_eval": 72.0}, + gguf_variants=[ + GGUFVariant( + filename="m-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={}, + task_profile="any", + ) + assert results + assert results[0].benchmark_status == "self_reported" + assert results[0].benchmark_source == "self_reported" + assert results[0].benchmark_confidence > 0.0 + + +def test_benchmark_source_and_confidence_exposed_for_none(): + model = ModelInfo( + id="someorg/unknown-7B", + family_id="unknown-7b", + name="unknown-7B", + parameter_count=7_000_000_000, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="u-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_000_000_000, + ), + ], + ) + hw = _make_hardware() + results = rank_models( + [model], + hw, + top_n=1, + benchmark_scores={}, + task_profile="any", + ) + assert results + assert results[0].benchmark_status == "none" + assert results[0].benchmark_source == "none" + assert results[0].benchmark_confidence == 0.0 + + +def test_ctx_penalty_demotes_non_fitting(): + models = [ + ModelInfo( + id="org/LongCtx-8B", + family_id="longctx-8b", + name="LongCtx-8B", + parameter_count=8_000_000_000, + context_length=131072, + downloads=900, + likes=90, + gguf_variants=[ + GGUFVariant( + filename="long-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + ], + ), + ModelInfo( + id="org/ShortCtx-8B", + family_id="shortctx-8b", + name="ShortCtx-8B", + parameter_count=8_000_000_000, + context_length=8192, + downloads=1000, + likes=100, + gguf_variants=[ + GGUFVariant( + filename="short-Q4_K_M.gguf", + quant_type="Q4_K_M", + file_size_bytes=4_500_000_000, + ), + ], + ), + ] + scores = { + "org/LongCtx-8B": 74.0, + "org/ShortCtx-8B": 76.0, + } + hw = _make_hardware(bandwidth_gbps=900.0) + + results = rank_models( + models, + hw, + context_length=32768, + top_n=2, + benchmark_scores=scores, + require_direct_top=False, + task_profile="any", + ) + + assert len(results) == 2 + assert results[0].model.family_id == "longctx-8b" + assert results[0].context_fits is True + assert results[1].context_fits is False diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..f68376c --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,81 @@ +"""Tests for shared utilities.""" + +from pathlib import Path + +import pytest + +from whichllm.utils import _cache_dir, parse_context_length, CONTEXT_LENGTH + + +def test_cache_dir_defaults_to_dot_cache(monkeypatch): + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + result = _cache_dir() + assert result == Path.home() / ".cache" / "whichllm" + + +def test_cache_dir_respects_xdg_cache_home(monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", "/tmp/custom-cache") + result = _cache_dir() + assert result == Path("/tmp/custom-cache/whichllm") + + +def test_cache_dir_falls_back_on_empty_xdg(monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", "") + result = _cache_dir() + assert result == Path.home() / ".cache" / "whichllm" + + +def test_cache_dir_ignores_relative_xdg(monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", "relative/path") + result = _cache_dir() + assert result == Path.home() / ".cache" / "whichllm" + + +# --- parse_context_length tests --- + + +def test_parse_plain_integer(): + assert parse_context_length("4096") == 4096 + assert parse_context_length("131072") == 131072 + + +def test_parse_k_suffix(): + assert parse_context_length("64k") == 64 * 1024 + assert parse_context_length("128K") == 128 * 1024 + + +def test_parse_m_suffix(): + assert parse_context_length("1m") == 1024 * 1024 + assert parse_context_length("2M") == 2 * 1024 * 1024 + + +def test_parse_fractional_suffix(): + assert parse_context_length("1.5k") == int(1.5 * 1024) + assert parse_context_length("0.5m") == int(0.5 * 1024 * 1024) + + +def test_parse_whitespace_is_stripped(): + assert parse_context_length(" 64k ") == 64 * 1024 + + +def test_parse_rejects_invalid_string(): + with pytest.raises(ValueError, match="Invalid context length"): + parse_context_length("abc") + + +def test_parse_rejects_zero(): + with pytest.raises(ValueError, match="must be positive"): + parse_context_length("0") + + +def test_parse_rejects_negative(): + with pytest.raises(ValueError, match="must be positive"): + parse_context_length("-1") + + +def test_click_type_passes_int_through(): + assert CONTEXT_LENGTH.convert(4096, None, None) == 4096 + + +def test_click_type_parses_shorthand(): + assert CONTEXT_LENGTH.convert("64k", None, None) == 64 * 1024 diff --git a/tests/test_vram.py b/tests/test_vram.py new file mode 100644 index 0000000..74b54fd --- /dev/null +++ b/tests/test_vram.py @@ -0,0 +1,148 @@ +"""Tests for VRAM estimation.""" + +from whichllm.engine.vram import estimate_kv_cache, estimate_vram +from whichllm.models.types import GGUFVariant, ModelInfo + + +def _make_model(params: int, **kwargs) -> ModelInfo: + return ModelInfo( + id="test/model", + family_id="test/model", + name="model", + parameter_count=params, + **kwargs, + ) + + +def test_estimate_vram_gguf_variant(): + model = _make_model(7_000_000_000) + variant = GGUFVariant( + filename="model-Q4_K_M.gguf", quant_type="Q4_K_M", file_size_bytes=4_000_000_000 + ) + vram = estimate_vram(model, variant, context_length=4096) + # Should be: 4GB weights + KV cache + activation + framework overhead + assert vram > 4_000_000_000 + assert vram < 7_000_000_000 # should be well under FP16 size + + +def test_estimate_vram_fp16_fallback(): + model = _make_model(7_000_000_000) + vram = estimate_vram(model, None, context_length=4096) + # FP16: 7B * 2 = 14GB + overhead + assert vram > 14_000_000_000 + assert vram < 20_000_000_000 + + +def test_estimate_vram_increases_with_context(): + model = _make_model(7_000_000_000) + variant = GGUFVariant( + filename="model-Q4_K_M.gguf", quant_type="Q4_K_M", file_size_bytes=4_000_000_000 + ) + vram_4k = estimate_vram(model, variant, context_length=4096) + vram_32k = estimate_vram(model, variant, context_length=32768) + assert vram_32k > vram_4k + + +def test_estimate_kv_cache_scales_with_params(): + small = _make_model(1_000_000_000) + large = _make_model(70_000_000_000) + kv_small = estimate_kv_cache(small, 4096) + kv_large = estimate_kv_cache(large, 4096) + assert kv_large > kv_small + + +def test_estimate_vram_small_model(): + model = _make_model(500_000_000) # 0.5B + variant = GGUFVariant( + filename="model-Q4_K_M.gguf", quant_type="Q4_K_M", file_size_bytes=300_000_000 + ) + vram = estimate_vram(model, variant, context_length=4096) + # Should be reasonable for a tiny model + assert vram > 300_000_000 + assert vram < 3_000_000_000 + + +def test_kv_cache_unchanged_when_no_sliding_window(): + """Models without an honored sliding window keep the full-context KV figure. + + This is the conservative-default guarantee: the SWA change must be a no-op + for every model that does not advertise an honored window. Expected values + are pinned literals (the current 3.5 MB/B/Kctx coefficient) so the test + fails if either the formula or the coefficient drifts. + """ + dense = _make_model(7_000_000_000) + expected = { + 4096: 102_760_448, + 32768: 822_083_584, + 131072: 3_288_334_336, + } + for ctx, want in expected.items(): + assert estimate_kv_cache(dense, ctx) == want + + +def test_kv_cache_mistral_window_in_config_is_ignored(): + """A declared window is NOT honored unless the architecture is allowlisted. + + Mistral-7B-v0.1 ships sliding_window=4096 in its config but mainline + runtimes ignore it, so whichllm must stay at full-context KV. We model this + by leaving sliding_window=None on the model; the estimate must equal dense. + """ + mistral = _make_model(7_000_000_000, architecture="mistral") + dense = _make_model(7_000_000_000) + for ctx in (4096, 131072): + assert estimate_kv_cache(mistral, ctx) == estimate_kv_cache(dense, ctx) + + +def test_kv_cache_pure_swa_plateaus_beyond_window(): + """Pure sliding-window models (global_ratio=0) plateau at the window size.""" + swa = _make_model( + 7_000_000_000, sliding_window=4096, sliding_window_global_ratio=0.0 + ) + at_window = estimate_kv_cache(swa, 4096) + far_beyond = estimate_kv_cache(swa, 131072) + # KV is flat once context exceeds the window. + assert far_beyond == at_window + # And it is far below the dense estimate at the same long context. + dense = estimate_kv_cache(_make_model(7_000_000_000), 131072) + assert far_beyond < dense / 10 + + +def test_kv_cache_hybrid_grows_slower_than_dense(): + """Hybrid SWA models grow with context but much slower than dense.""" + # Gemma-3-like: 1/6 global layers, 1024-token window. + hybrid = _make_model( + 27_000_000_000, + sliding_window=1024, + sliding_window_global_ratio=1.0 / 6.0, + ) + dense = _make_model(27_000_000_000) + short = estimate_kv_cache(hybrid, 4096) + long_hybrid = estimate_kv_cache(hybrid, 131072) + long_dense = estimate_kv_cache(dense, 131072) + # Still grows with context (global layers keep scaling)... + assert long_hybrid > short + # ...but well under the dense estimate (roughly the global ratio). + assert long_hybrid < long_dense + assert long_hybrid < long_dense * 0.30 + + +def test_kv_cache_never_exceeds_dense_estimate(): + """The SWA reduction can only ever lower the estimate, never raise it.""" + for ratio in (0.0, 1.0 / 6.0, 0.25, 0.5, 1.0): + swa = _make_model( + 13_000_000_000, + sliding_window=2048, + sliding_window_global_ratio=ratio, + ) + dense = _make_model(13_000_000_000) + for ctx in (1024, 4096, 65536): + assert estimate_kv_cache(swa, ctx) <= estimate_kv_cache(dense, ctx) + + +def test_kv_cache_below_window_matches_dense(): + """When context fits inside the window, SWA and dense agree.""" + swa = _make_model( + 7_000_000_000, sliding_window=8192, sliding_window_global_ratio=0.0 + ) + dense = _make_model(7_000_000_000) + assert estimate_kv_cache(swa, 4096) == estimate_kv_cache(dense, 4096) diff --git a/tests/test_windows_cpu_detection.py b/tests/test_windows_cpu_detection.py new file mode 100644 index 0000000..c9f35ea --- /dev/null +++ b/tests/test_windows_cpu_detection.py @@ -0,0 +1,45 @@ +"""Tests for Windows CPU name fallback detection.""" + +from __future__ import annotations + +import subprocess + +from whichllm.hardware import cpu + + +def test_windows_cpu_name_uses_cim_when_wmic_is_empty(monkeypatch): + monkeypatch.setattr(cpu.platform, "system", lambda: "Windows") + + def fake_run(args, **kwargs): + if args == ["wmic", "cpu", "get", "name"]: + return subprocess.CompletedProcess(args, 0, stdout="Name\n\n", stderr="") + if args[0] == "powershell": + return subprocess.CompletedProcess( + args, + 0, + stdout="Intel(R) Xeon(R) W-11955M CPU @ 2.60GHz\n", + stderr="", + ) + raise FileNotFoundError + + monkeypatch.setattr(cpu.subprocess, "run", fake_run) + + assert cpu.detect_cpu_name() == "Intel(R) Xeon(R) W-11955M CPU @ 2.60GHz" + + +def test_windows_cpu_name_uses_wmic_when_available(monkeypatch): + monkeypatch.setattr(cpu.platform, "system", lambda: "Windows") + + def fake_run(args, **kwargs): + if args == ["wmic", "cpu", "get", "name"]: + return subprocess.CompletedProcess( + args, + 0, + stdout="Name\nIntel(R) Core(TM) i9-11900H @ 2.50GHz\n", + stderr="", + ) + raise AssertionError("CIM fallback should not run when wmic returns a name") + + monkeypatch.setattr(cpu.subprocess, "run", fake_run) + + assert cpu.detect_cpu_name() == "Intel(R) Core(TM) i9-11900H @ 2.50GHz" diff --git a/tests/test_windows_gpu.py b/tests/test_windows_gpu.py new file mode 100644 index 0000000..1fe3b86 --- /dev/null +++ b/tests/test_windows_gpu.py @@ -0,0 +1,175 @@ +"""Tests for Windows GPU detection.""" + +from __future__ import annotations + +import json +import subprocess + +from whichllm.hardware import windows + + +def test_detect_windows_amd_gpu(monkeypatch): + output = json.dumps( + { + "Name": "AMD Radeon RX 9060 XT", + "AdapterRAM": 16 * 1024**3, + } + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "AMD Radeon RX 9060 XT" + assert gpus[0].vendor == "amd" + assert gpus[0].vram_bytes == 16 * 1024**3 + assert gpus[0].shared_memory is False + assert gpus[0].memory_bandwidth_gbps == 320.0 + + +def test_detect_windows_amd_gpu_prefers_64_bit_dedicated_memory(monkeypatch): + output = json.dumps( + { + "Name": "AMD Radeon RX 9060 XT", + "AdapterRAM": 4 * 1024**3, + "DedicatedVideoMemory": 16 * 1024**3, + } + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].vram_bytes == 16 * 1024**3 + assert gpus[0].shared_memory is False + + +def test_detect_windows_gpu_queries_control_video_qword_memory(monkeypatch): + captured: dict[str, str] = {} + + def fake_run(*args, **kwargs): + command = args[0] + captured["script"] = command[-1] + return subprocess.CompletedProcess(command, 0, stdout="[]", stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + assert windows.detect_windows_gpus() == [] + assert "Control\\Video" in captured["script"] + assert "VideoID" in captured["script"] + assert "HardwareInformation.qwMemorySize" in captured["script"] + + +def test_detect_windows_amd_gpu_applies_known_floor_for_capped_adapter_ram( + monkeypatch, +): + output = json.dumps( + { + "Name": "AMD Radeon RX 9060 XT", + "AdapterRAM": 4 * 1024**3, + "DedicatedVideoMemory": None, + } + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].vram_bytes == 8 * 1024**3 + assert gpus[0].shared_memory is False + + +def test_detect_windows_ryzen_ai_radeon_890m_as_shared_memory(monkeypatch): + output = json.dumps( + { + "Name": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M", + "AdapterRAM": 512 * 1024**2, + "DedicatedVideoMemory": None, + } + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "AMD Ryzen AI 9 HX 370 w/ Radeon 890M" + assert gpus[0].vendor == "amd" + assert gpus[0].vram_bytes == 0 + assert gpus[0].shared_memory is True + assert gpus[0].memory_bandwidth_gbps == 120.0 + + +def test_detect_windows_intel_gpu_list(monkeypatch): + output = json.dumps( + [ + { + "Name": "Intel(R) Arc(TM) Graphics", + "AdapterRAM": None, + }, + { + "Name": "Microsoft Basic Display Adapter", + "AdapterRAM": 0, + }, + ] + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "Intel(R) Arc(TM) Graphics" + assert gpus[0].vendor == "intel" + assert gpus[0].vram_bytes == 0 + assert gpus[0].shared_memory is True + + +def test_detect_windows_intel_arc_discrete_is_not_shared_memory(monkeypatch): + output = json.dumps( + { + "Name": "Intel(R) Arc(TM) B580 Graphics", + "AdapterRAM": 4 * 1024**3, + "DedicatedVideoMemory": 12 * 1024**3, + } + ) + + def fake_run(*args, **kwargs): + return subprocess.CompletedProcess(args[0], 0, stdout=output, stderr="") + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + gpus = windows.detect_windows_gpus() + + assert len(gpus) == 1 + assert gpus[0].name == "Intel(R) Arc(TM) B580 Graphics" + assert gpus[0].vendor == "intel" + assert gpus[0].vram_bytes == 12 * 1024**3 + assert gpus[0].shared_memory is False + + +def test_detect_windows_gpu_returns_empty_on_command_failure(monkeypatch): + def fake_run(*args, **kwargs): + raise FileNotFoundError + + monkeypatch.setattr(windows.subprocess, "run", fake_run) + + assert windows.detect_windows_gpus() == [] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..da5cc41 --- /dev/null +++ b/uv.lock @@ -0,0 +1,581 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, +] + +[[package]] +name = "certifi" +version = "2026.2.25" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dbgpu" +version = "2025.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/89/3f020bdd2236ba974f0c4f1b31ce67c54bd127ca0d2296ef48b8e94c3bc3/dbgpu-2025.12.tar.gz", hash = "sha256:d4a2fdc36ff5ff2af37e8fd8a3e0740ab2af73cc5e0fda199fdff3d6f1686f4e", size = 225653, upload-time = "2025-12-15T16:57:20.958Z" } + +[package.optional-dependencies] +fuzz = [ + { name = "thefuzz" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "nvidia-ml-py" +version = "13.590.48" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/a0/f4fc18cf72f06821a9a665085435b901449986855519d5b3843532db35e9/nvidia_ml_py-13.590.48.tar.gz", hash = "sha256:8184d1be52914ac7f0991cd1c0d946c65dc88a840c754cd12c274b77b88760dd", size = 49732, upload-time = "2026-01-22T01:14:56.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/72/fb2af0d259a651affdce65fd6a495f0e07a685a0136baf585c5065204ee7/nvidia_ml_py-13.590.48-py3-none-any.whl", hash = "sha256:fd43d30ee9cd0b7940f5f9f9220b68d42722975e3992b6c21d14144c48760e43", size = 50680, upload-time = "2026-01-22T01:14:55.281Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "pyarrow" +version = "23.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, + { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, + { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, + { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, + { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" }, + { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" }, + { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" }, + { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" }, + { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "rapidfuzz" +version = "3.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/28/9d808fe62375b9aab5ba92fa9b29371297b067c2790b2d7cda648b1e2f8d/rapidfuzz-3.14.3.tar.gz", hash = "sha256:2491937177868bc4b1e469087601d53f925e8d270ccc21e07404b4b5814b7b5f", size = 57863900, upload-time = "2025-11-01T11:54:52.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/25/5b0a33ad3332ee1213068c66f7c14e9e221be90bab434f0cb4defa9d6660/rapidfuzz-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dea2d113e260a5da0c4003e0a5e9fdf24a9dc2bb9eaa43abd030a1e46ce7837d", size = 1953885, upload-time = "2025-11-01T11:52:47.75Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ab/f1181f500c32c8fcf7c966f5920c7e56b9b1d03193386d19c956505c312d/rapidfuzz-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e6c31a4aa68cfa75d7eede8b0ed24b9e458447db604c2db53f358be9843d81d3", size = 1390200, upload-time = "2025-11-01T11:52:49.491Z" }, + { url = "https://files.pythonhosted.org/packages/14/2a/0f2de974ececad873865c6bb3ea3ad07c976ac293d5025b2d73325aac1d4/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02821366d928e68ddcb567fed8723dad7ea3a979fada6283e6914d5858674850", size = 1389319, upload-time = "2025-11-01T11:52:51.224Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/309d8f3a0bb3031fd9b667174cc4af56000645298af7c2931be5c3d14bb4/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfe8df315ab4e6db4e1be72c5170f8e66021acde22cd2f9d04d2058a9fd8162e", size = 3178495, upload-time = "2025-11-01T11:52:53.005Z" }, + { url = "https://files.pythonhosted.org/packages/10/b7/f9c44a99269ea5bf6fd6a40b84e858414b6e241288b9f2b74af470d222b1/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:769f31c60cd79420188fcdb3c823227fc4a6deb35cafec9d14045c7f6743acae", size = 1228443, upload-time = "2025-11-01T11:52:54.991Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0a/3b3137abac7f19c9220e14cd7ce993e35071a7655e7ef697785a3edfea1a/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54fa03062124e73086dae66a3451c553c1e20a39c077fd704dc7154092c34c63", size = 2411998, upload-time = "2025-11-01T11:52:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b6/983805a844d44670eaae63831024cdc97ada4e9c62abc6b20703e81e7f9b/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:834d1e818005ed0d4ae38f6b87b86fad9b0a74085467ece0727d20e15077c094", size = 2530120, upload-time = "2025-11-01T11:52:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cc/2c97beb2b1be2d7595d805682472f1b1b844111027d5ad89b65e16bdbaaa/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:948b00e8476a91f510dd1ec07272efc7d78c275d83b630455559671d4e33b678", size = 4283129, upload-time = "2025-11-01T11:53:00.188Z" }, + { url = "https://files.pythonhosted.org/packages/4d/03/2f0e5e94941045aefe7eafab72320e61285c07b752df9884ce88d6b8b835/rapidfuzz-3.14.3-cp311-cp311-win32.whl", hash = "sha256:43d0305c36f504232f18ea04e55f2059bb89f169d3119c4ea96a0e15b59e2a91", size = 1724224, upload-time = "2025-11-01T11:53:02.149Z" }, + { url = "https://files.pythonhosted.org/packages/cf/99/5fa23e204435803875daefda73fd61baeabc3c36b8fc0e34c1705aab8c7b/rapidfuzz-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:ef6bf930b947bd0735c550683939a032090f1d688dfd8861d6b45307b96fd5c5", size = 1544259, upload-time = "2025-11-01T11:53:03.66Z" }, + { url = "https://files.pythonhosted.org/packages/48/35/d657b85fcc615a42661b98ac90ce8e95bd32af474603a105643963749886/rapidfuzz-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:f3eb0ff3b75d6fdccd40b55e7414bb859a1cda77c52762c9c82b85569f5088e7", size = 814734, upload-time = "2025-11-01T11:53:05.008Z" }, + { url = "https://files.pythonhosted.org/packages/fa/8e/3c215e860b458cfbedb3ed73bc72e98eb7e0ed72f6b48099604a7a3260c2/rapidfuzz-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:685c93ea961d135893b5984a5a9851637d23767feabe414ec974f43babbd8226", size = 1945306, upload-time = "2025-11-01T11:53:06.452Z" }, + { url = "https://files.pythonhosted.org/packages/36/d9/31b33512015c899f4a6e6af64df8dfe8acddf4c8b40a4b3e0e6e1bcd00e5/rapidfuzz-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fa7c8f26f009f8c673fbfb443792f0cf8cf50c4e18121ff1e285b5e08a94fbdb", size = 1390788, upload-time = "2025-11-01T11:53:08.721Z" }, + { url = "https://files.pythonhosted.org/packages/a9/67/2ee6f8de6e2081ccd560a571d9c9063184fe467f484a17fa90311a7f4a2e/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57f878330c8d361b2ce76cebb8e3e1dc827293b6abf404e67d53260d27b5d941", size = 1374580, upload-time = "2025-11-01T11:53:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/30/83/80d22997acd928eda7deadc19ccd15883904622396d6571e935993e0453a/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c5f545f454871e6af05753a0172849c82feaf0f521c5ca62ba09e1b382d6382", size = 3154947, upload-time = "2025-11-01T11:53:12.093Z" }, + { url = "https://files.pythonhosted.org/packages/5b/cf/9f49831085a16384695f9fb096b99662f589e30b89b4a589a1ebc1a19d34/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:07aa0b5d8863e3151e05026a28e0d924accf0a7a3b605da978f0359bb804df43", size = 1223872, upload-time = "2025-11-01T11:53:13.664Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/41ee8034e744b871c2e071ef0d360686f5ccfe5659f4fd96c3ec406b3c8b/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73b07566bc7e010e7b5bd490fb04bb312e820970180df6b5655e9e6224c137db", size = 2392512, upload-time = "2025-11-01T11:53:15.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/86/280038b6b0c2ccec54fb957c732ad6b41cc1fd03b288d76545b9cf98343f/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6de00eb84c71476af7d3110cf25d8fe7c792d7f5fa86764ef0b4ca97e78ca3ed", size = 2521398, upload-time = "2025-11-01T11:53:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7b/05c26f939607dca0006505e3216248ae2de631e39ef94dd63dbbf0860021/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d7843a1abf0091773a530636fdd2a49a41bcae22f9910b86b4f903e76ddc82dc", size = 4259416, upload-time = "2025-11-01T11:53:19.34Z" }, + { url = "https://files.pythonhosted.org/packages/40/eb/9e3af4103d91788f81111af1b54a28de347cdbed8eaa6c91d5e98a889aab/rapidfuzz-3.14.3-cp312-cp312-win32.whl", hash = "sha256:dea97ac3ca18cd3ba8f3d04b5c1fe4aa60e58e8d9b7793d3bd595fdb04128d7a", size = 1709527, upload-time = "2025-11-01T11:53:20.949Z" }, + { url = "https://files.pythonhosted.org/packages/b8/63/d06ecce90e2cf1747e29aeab9f823d21e5877a4c51b79720b2d3be7848f8/rapidfuzz-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:b5100fd6bcee4d27f28f4e0a1c6b5127bc8ba7c2a9959cad9eab0bf4a7ab3329", size = 1538989, upload-time = "2025-11-01T11:53:22.428Z" }, + { url = "https://files.pythonhosted.org/packages/fc/6d/beee32dcda64af8128aab3ace2ccb33d797ed58c434c6419eea015fec779/rapidfuzz-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:4e49c9e992bc5fc873bd0fff7ef16a4405130ec42f2ce3d2b735ba5d3d4eb70f", size = 811161, upload-time = "2025-11-01T11:53:23.811Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/0d94d09646853bd26978cb3a7541b6233c5760687777fa97da8de0d9a6ac/rapidfuzz-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dbcb726064b12f356bf10fffdb6db4b6dce5390b23627c08652b3f6e49aa56ae", size = 1939646, upload-time = "2025-11-01T11:53:25.292Z" }, + { url = "https://files.pythonhosted.org/packages/b6/eb/f96aefc00f3bbdbab9c0657363ea8437a207d7545ac1c3789673e05d80bd/rapidfuzz-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1704fc70d214294e554a2421b473779bcdeef715881c5e927dc0f11e1692a0ff", size = 1385512, upload-time = "2025-11-01T11:53:27.594Z" }, + { url = "https://files.pythonhosted.org/packages/26/34/71c4f7749c12ee223dba90017a5947e8f03731a7cc9f489b662a8e9e643d/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc65e72790ddfd310c2c8912b45106e3800fefe160b0c2ef4d6b6fec4e826457", size = 1373571, upload-time = "2025-11-01T11:53:29.096Z" }, + { url = "https://files.pythonhosted.org/packages/32/00/ec8597a64f2be301ce1ee3290d067f49f6a7afb226b67d5f15b56d772ba5/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e38c1305cffae8472572a0584d4ffc2f130865586a81038ca3965301f7c97c", size = 3156759, upload-time = "2025-11-01T11:53:30.777Z" }, + { url = "https://files.pythonhosted.org/packages/61/d5/b41eeb4930501cc899d5a9a7b5c9a33d85a670200d7e81658626dcc0ecc0/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:e195a77d06c03c98b3fc06b8a28576ba824392ce40de8c708f96ce04849a052e", size = 1222067, upload-time = "2025-11-01T11:53:32.334Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7d/6d9abb4ffd1027c6ed837b425834f3bed8344472eb3a503ab55b3407c721/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b7ef2f4b8583a744338a18f12c69693c194fb6777c0e9ada98cd4d9e8f09d10", size = 2394775, upload-time = "2025-11-01T11:53:34.24Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/4f3ab4c401c5a55364da1ffff8cc879fc97b4e5f4fa96033827da491a973/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a2135b138bcdcb4c3742d417f215ac2d8c2b87bde15b0feede231ae95f09ec41", size = 2526123, upload-time = "2025-11-01T11:53:35.779Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4b/54f804975376a328f57293bd817c12c9036171d15cf7292032e3f5820b2d/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33a325ed0e8e1aa20c3e75f8ab057a7b248fdea7843c2a19ade0008906c14af0", size = 4262874, upload-time = "2025-11-01T11:53:37.866Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b6/958db27d8a29a50ee6edd45d33debd3ce732e7209183a72f57544cd5fe22/rapidfuzz-3.14.3-cp313-cp313-win32.whl", hash = "sha256:8383b6d0d92f6cd008f3c9216535be215a064b2cc890398a678b56e6d280cb63", size = 1707972, upload-time = "2025-11-01T11:53:39.442Z" }, + { url = "https://files.pythonhosted.org/packages/07/75/fde1f334b0cec15b5946d9f84d73250fbfcc73c236b4bc1b25129d90876b/rapidfuzz-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:e6b5e3036976f0fde888687d91be86d81f9ac5f7b02e218913c38285b756be6c", size = 1537011, upload-time = "2025-11-01T11:53:40.92Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d7/d83fe001ce599dc7ead57ba1debf923dc961b6bdce522b741e6b8c82f55c/rapidfuzz-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:7ba009977601d8b0828bfac9a110b195b3e4e79b350dcfa48c11269a9f1918a0", size = 810744, upload-time = "2025-11-01T11:53:42.723Z" }, + { url = "https://files.pythonhosted.org/packages/92/13/a486369e63ff3c1a58444d16b15c5feb943edd0e6c28a1d7d67cb8946b8f/rapidfuzz-3.14.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a0a28add871425c2fe94358c6300bbeb0bc2ed828ca003420ac6825408f5a424", size = 1967702, upload-time = "2025-11-01T11:53:44.554Z" }, + { url = "https://files.pythonhosted.org/packages/f1/82/efad25e260b7810f01d6b69122685e355bed78c94a12784bac4e0beb2afb/rapidfuzz-3.14.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:010e12e2411a4854b0434f920e72b717c43f8ec48d57e7affe5c42ecfa05dd0e", size = 1410702, upload-time = "2025-11-01T11:53:46.066Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1a/34c977b860cde91082eae4a97ae503f43e0d84d4af301d857679b66f9869/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cfc3d57abd83c734d1714ec39c88a34dd69c85474918ebc21296f1e61eb5ca8", size = 1382337, upload-time = "2025-11-01T11:53:47.62Z" }, + { url = "https://files.pythonhosted.org/packages/88/74/f50ea0e24a5880a9159e8fd256b84d8f4634c2f6b4f98028bdd31891d907/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89acb8cbb52904f763e5ac238083b9fc193bed8d1f03c80568b20e4cef43a519", size = 3165563, upload-time = "2025-11-01T11:53:49.216Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7a/e744359404d7737049c26099423fc54bcbf303de5d870d07d2fb1410f567/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_31_armv7l.whl", hash = "sha256:7d9af908c2f371bfb9c985bd134e295038e3031e666e4b2ade1e7cb7f5af2f1a", size = 1214727, upload-time = "2025-11-01T11:53:50.883Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2e/87adfe14ce75768ec6c2b8acd0e05e85e84be4be5e3d283cdae360afc4fe/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1f1925619627f8798f8c3a391d81071336942e5fe8467bc3c567f982e7ce2897", size = 2403349, upload-time = "2025-11-01T11:53:52.322Z" }, + { url = "https://files.pythonhosted.org/packages/70/17/6c0b2b2bff9c8b12e12624c07aa22e922b0c72a490f180fa9183d1ef2c75/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:152555187360978119e98ce3e8263d70dd0c40c7541193fc302e9b7125cf8f58", size = 2507596, upload-time = "2025-11-01T11:53:53.835Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d1/87852a7cbe4da7b962174c749a47433881a63a817d04f3e385ea9babcd9e/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52619d25a09546b8db078981ca88939d72caa6b8701edd8b22e16482a38e799f", size = 4273595, upload-time = "2025-11-01T11:53:55.961Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ab/1d0354b7d1771a28fa7fe089bc23acec2bdd3756efa2419f463e3ed80e16/rapidfuzz-3.14.3-cp313-cp313t-win32.whl", hash = "sha256:489ce98a895c98cad284f0a47960c3e264c724cb4cfd47a1430fa091c0c25204", size = 1757773, upload-time = "2025-11-01T11:53:57.628Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0c/71ef356adc29e2bdf74cd284317b34a16b80258fa0e7e242dd92cc1e6d10/rapidfuzz-3.14.3-cp313-cp313t-win_amd64.whl", hash = "sha256:656e52b054d5b5c2524169240e50cfa080b04b1c613c5f90a2465e84888d6f15", size = 1576797, upload-time = "2025-11-01T11:53:59.455Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d2/0e64fc27bb08d4304aa3d11154eb5480bcf5d62d60140a7ee984dc07468a/rapidfuzz-3.14.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c7e40c0a0af02ad6e57e89f62bef8604f55a04ecae90b0ceeda591bbf5923317", size = 829940, upload-time = "2025-11-01T11:54:01.1Z" }, + { url = "https://files.pythonhosted.org/packages/32/6f/1b88aaeade83abc5418788f9e6b01efefcd1a69d65ded37d89cd1662be41/rapidfuzz-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:442125473b247227d3f2de807a11da6c08ccf536572d1be943f8e262bae7e4ea", size = 1942086, upload-time = "2025-11-01T11:54:02.592Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2c/b23861347436cb10f46c2bd425489ec462790faaa360a54a7ede5f78de88/rapidfuzz-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ec0c8c0c3d4f97ced46b2e191e883f8c82dbbf6d5ebc1842366d7eff13cd5a6", size = 1386993, upload-time = "2025-11-01T11:54:04.12Z" }, + { url = "https://files.pythonhosted.org/packages/83/86/5d72e2c060aa1fbdc1f7362d938f6b237dff91f5b9fc5dd7cc297e112250/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2dc37bc20272f388b8c3a4eba4febc6e77e50a8f450c472def4751e7678f55e4", size = 1379126, upload-time = "2025-11-01T11:54:05.777Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/ef2cee3e4d8b3fc22705ff519f0d487eecc756abdc7c25d53686689d6cf2/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dee362e7e79bae940a5e2b3f6d09c6554db6a4e301cc68343886c08be99844f1", size = 3159304, upload-time = "2025-11-01T11:54:07.351Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/dc5f2f62bbc7bc90be1f75eeaf49ed9502094bb19290dfb4747317b17f12/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:4b39921df948388a863f0e267edf2c36302983459b021ab928d4b801cbe6a421", size = 1218207, upload-time = "2025-11-01T11:54:09.641Z" }, + { url = "https://files.pythonhosted.org/packages/df/7e/8f4be75c1bc62f47edf2bbbe2370ee482fae655ebcc4718ac3827ead3904/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:beda6aa9bc44d1d81242e7b291b446be352d3451f8217fcb068fc2933927d53b", size = 2401245, upload-time = "2025-11-01T11:54:11.543Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/f7c92759e1bb188dd05b80d11c630ba59b8d7856657baf454ff56059c2ab/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6a014ba09657abfcfeed64b7d09407acb29af436d7fc075b23a298a7e4a6b41c", size = 2518308, upload-time = "2025-11-01T11:54:13.134Z" }, + { url = "https://files.pythonhosted.org/packages/c7/ac/85820f70fed5ecb5f1d9a55f1e1e2090ef62985ef41db289b5ac5ec56e28/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:32eeafa3abce138bb725550c0e228fc7eaeec7059aa8093d9cbbec2b58c2371a", size = 4265011, upload-time = "2025-11-01T11:54:15.087Z" }, + { url = "https://files.pythonhosted.org/packages/46/a9/616930721ea9835c918af7cde22bff17f9db3639b0c1a7f96684be7f5630/rapidfuzz-3.14.3-cp314-cp314-win32.whl", hash = "sha256:adb44d996fc610c7da8c5048775b21db60dd63b1548f078e95858c05c86876a3", size = 1742245, upload-time = "2025-11-01T11:54:17.19Z" }, + { url = "https://files.pythonhosted.org/packages/06/8a/f2fa5e9635b1ccafda4accf0e38246003f69982d7c81f2faa150014525a4/rapidfuzz-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:f3d15d8527e2b293e38ce6e437631af0708df29eafd7c9fc48210854c94472f9", size = 1584856, upload-time = "2025-11-01T11:54:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/ef/97/09e20663917678a6d60d8e0e29796db175b1165e2079830430342d5298be/rapidfuzz-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:576e4b9012a67e0bf54fccb69a7b6c94d4e86a9540a62f1a5144977359133583", size = 833490, upload-time = "2025-11-01T11:54:20.753Z" }, + { url = "https://files.pythonhosted.org/packages/03/1b/6b6084576ba87bf21877c77218a0c97ba98cb285b0c02eaaee3acd7c4513/rapidfuzz-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cec3c0da88562727dd5a5a364bd9efeb535400ff0bfb1443156dd139a1dd7b50", size = 1968658, upload-time = "2025-11-01T11:54:22.25Z" }, + { url = "https://files.pythonhosted.org/packages/38/c0/fb02a0db80d95704b0a6469cc394e8c38501abf7e1c0b2afe3261d1510c2/rapidfuzz-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d1fa009f8b1100e4880868137e7bf0501422898f7674f2adcd85d5a67f041296", size = 1410742, upload-time = "2025-11-01T11:54:23.863Z" }, + { url = "https://files.pythonhosted.org/packages/a4/72/3fbf12819fc6afc8ec75a45204013b40979d068971e535a7f3512b05e765/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b86daa7419b5e8b180690efd1fdbac43ff19230803282521c5b5a9c83977655", size = 1382810, upload-time = "2025-11-01T11:54:25.571Z" }, + { url = "https://files.pythonhosted.org/packages/0f/18/0f1991d59bb7eee28922a00f79d83eafa8c7bfb4e8edebf4af2a160e7196/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7bd1816db05d6c5ffb3a4df0a2b7b56fb8c81ef584d08e37058afa217da91b1", size = 3166349, upload-time = "2025-11-01T11:54:27.195Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f0/baa958b1989c8f88c78bbb329e969440cf330b5a01a982669986495bb980/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:33da4bbaf44e9755b0ce192597f3bde7372fe2e381ab305f41b707a95ac57aa7", size = 1214994, upload-time = "2025-11-01T11:54:28.821Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a0/cd12ec71f9b2519a3954febc5740291cceabc64c87bc6433afcb36259f3b/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3fecce764cf5a991ee2195a844196da840aba72029b2612f95ac68a8b74946bf", size = 2403919, upload-time = "2025-11-01T11:54:30.393Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ce/019bd2176c1644098eced4f0595cb4b3ef52e4941ac9a5854f209d0a6e16/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ecd7453e02cf072258c3a6b8e930230d789d5d46cc849503729f9ce475d0e785", size = 2508346, upload-time = "2025-11-01T11:54:32.048Z" }, + { url = "https://files.pythonhosted.org/packages/23/f8/be16c68e2c9e6c4f23e8f4adbb7bccc9483200087ed28ff76c5312da9b14/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea188aa00e9bcae8c8411f006a5f2f06c4607a02f24eab0d8dc58566aa911f35", size = 4274105, upload-time = "2025-11-01T11:54:33.701Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d1/5ab148e03f7e6ec8cd220ccf7af74d3aaa4de26dd96df58936beb7cba820/rapidfuzz-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:7ccbf68100c170e9a0581accbe9291850936711548c6688ce3bfb897b8c589ad", size = 1793465, upload-time = "2025-11-01T11:54:35.331Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/433b2d98e97abd9fff1c470a109b311669f44cdec8d0d5aa250aceaed1fb/rapidfuzz-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9ec02e62ae765a318d6de38df609c57fc6dacc65c0ed1fd489036834fd8a620c", size = 1623491, upload-time = "2025-11-01T11:54:38.085Z" }, + { url = "https://files.pythonhosted.org/packages/e2/f6/e2176eb94f94892441bce3ddc514c179facb65db245e7ce3356965595b19/rapidfuzz-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:e805e52322ae29aa945baf7168b6c898120fbc16d2b8f940b658a5e9e3999253", size = 851487, upload-time = "2025-11-01T11:54:40.176Z" }, + { url = "https://files.pythonhosted.org/packages/c9/33/b5bd6475c7c27164b5becc9b0e3eb978f1e3640fea590dd3dced6006ee83/rapidfuzz-3.14.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7cf174b52cb3ef5d49e45d0a1133b7e7d0ecf770ed01f97ae9962c5c91d97d23", size = 1888499, upload-time = "2025-11-01T11:54:42.094Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/89d65d4db4bb931beade9121bc71ad916b5fa9396e807d11b33731494e8e/rapidfuzz-3.14.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:442cba39957a008dfc5bdef21a9c3f4379e30ffb4e41b8555dbaf4887eca9300", size = 1336747, upload-time = "2025-11-01T11:54:43.957Z" }, + { url = "https://files.pythonhosted.org/packages/85/33/cd87d92b23f0b06e8914a61cea6850c6d495ca027f669fab7a379041827a/rapidfuzz-3.14.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1faa0f8f76ba75fd7b142c984947c280ef6558b5067af2ae9b8729b0a0f99ede", size = 1352187, upload-time = "2025-11-01T11:54:45.518Z" }, + { url = "https://files.pythonhosted.org/packages/22/20/9d30b4a1ab26aac22fff17d21dec7e9089ccddfe25151d0a8bb57001dc3d/rapidfuzz-3.14.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e6eefec45625c634926a9fd46c9e4f31118ac8f3156fff9494422cee45207e6", size = 3101472, upload-time = "2025-11-01T11:54:47.255Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ad/fa2d3e5c29a04ead7eaa731c7cd1f30f9ec3c77b3a578fdf90280797cbcb/rapidfuzz-3.14.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56fefb4382bb12250f164250240b9dd7772e41c5c8ae976fd598a32292449cc5", size = 1511361, upload-time = "2025-11-01T11:54:49.057Z" }, +] + +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "thefuzz" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/4b/d3eb25831590d6d7d38c2f2e3561d3ba41d490dc89cd91d9e65e7c812508/thefuzz-0.22.1.tar.gz", hash = "sha256:7138039a7ecf540da323792d8592ef9902b1d79eb78c147d4f20664de79f3680", size = 19993, upload-time = "2024-01-19T19:18:23.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/4f/1695e70ceb3604f19eda9908e289c687ea81c4fecef4d90a9d1d0f2f7ae9/thefuzz-0.22.1-py3-none-any.whl", hash = "sha256:59729b33556850b90e1093c4cf9e618af6f2e4c985df193fdf3c5b5cf02ca481", size = 8245, upload-time = "2024-01-19T19:18:20.362Z" }, +] + +[[package]] +name = "typer" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/24/cb09efec5cc954f7f9b930bf8279447d24618bb6758d4f6adf2574c41780/typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45", size = 118613, upload-time = "2026-02-21T16:54:40.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "whichllm" +version = "0.5.15" +source = { editable = "." } +dependencies = [ + { name = "dbgpu", extra = ["fuzz"] }, + { name = "httpx" }, + { name = "nvidia-ml-py" }, + { name = "psutil" }, + { name = "rich" }, + { name = "typer" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pyarrow" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "dbgpu", extras = ["fuzz"], specifier = ">=1.0" }, + { name = "httpx", specifier = ">=0.27" }, + { name = "nvidia-ml-py", specifier = ">=12.0" }, + { name = "psutil", specifier = ">=5.9" }, + { name = "rich", specifier = ">=13.0" }, + { name = "typer", specifier = ">=0.9" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pyarrow", specifier = ">=23.0.1" }, + { name = "pytest", specifier = ">=8.0" }, +]