chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,38 @@
|
||||
# Build artifacts
|
||||
target/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.iml
|
||||
|
||||
# Git
|
||||
.git/
|
||||
.gitignore
|
||||
.githooks/
|
||||
|
||||
# Documentation and assets
|
||||
*.md
|
||||
!README.md
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
.github/
|
||||
|
||||
# Scripts (not needed in container)
|
||||
scripts/
|
||||
install.sh
|
||||
|
||||
# Website files
|
||||
index.html
|
||||
CNAME
|
||||
|
||||
# Release metadata
|
||||
.release-please-manifest.json
|
||||
|
||||
# Skills (OpenClaw integration, not needed in container)
|
||||
skills/
|
||||
AGENTS.md
|
||||
@@ -0,0 +1 @@
|
||||
* text=auto eol=lf
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
echo "Running cargo fmt --check..."
|
||||
cargo fmt --check
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ Formatting issues found. Run 'cargo fmt' to fix."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Formatting OK"
|
||||
@@ -0,0 +1,129 @@
|
||||
name: Bug Report
|
||||
description: Report a bug in llmfit (incorrect fit analysis, hardware detection issues, TUI rendering problems, etc.)
|
||||
title: "[Bug]:(short issue description)"
|
||||
labels: ["bug", "needs-triage"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting a bug! Please fill in the details below so we can reproduce and fix it.
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Bug description
|
||||
description: A clear and concise description of what the bug is.
|
||||
placeholder: "e.g., llmfit reports 'Perfect' fit for a 70B model on 8GB RAM"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
description: What did you expect to happen?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: actual
|
||||
attributes:
|
||||
label: Actual behavior
|
||||
description: What actually happened instead?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: steps
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
description: How can we reproduce this issue?
|
||||
placeholder: |
|
||||
1. Run `llmfit --cli`
|
||||
2. Look at model X
|
||||
3. See incorrect fit level
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: doctor-output
|
||||
attributes:
|
||||
label: Diagnostic report (`llmfit doctor`)
|
||||
description: >
|
||||
Paste the full output of `llmfit doctor` (v0.9.36+). It captures your
|
||||
detected specs plus the raw nvidia-smi/rocm-smi/sysfs output detection
|
||||
relies on — for hardware bugs this is usually all we need to reproduce
|
||||
and write a regression test. On older versions paste `llmfit system`
|
||||
instead.
|
||||
render: text
|
||||
placeholder: Output of `llmfit doctor` (or `llmfit system` on older versions)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: llmfit version
|
||||
description: Output of `llmfit --version`
|
||||
placeholder: "llmfit 0.x.x"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
options:
|
||||
- Linux
|
||||
- macOS (Intel)
|
||||
- macOS (Apple Silicon)
|
||||
- Windows
|
||||
- WSL
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: component
|
||||
attributes:
|
||||
label: Affected component
|
||||
description: Which part of llmfit is affected?
|
||||
options:
|
||||
- Hardware detection (RAM/CPU/GPU)
|
||||
- Model fit analysis (scoring, fit levels)
|
||||
- TUI (rendering, navigation, keybindings)
|
||||
- CLI output (subcommands, table display)
|
||||
- Model database (missing/incorrect model data)
|
||||
- Download manager
|
||||
- Speed estimation (tok/s)
|
||||
- Provider detection (Ollama, llama.cpp, MLX, Docker)
|
||||
- MCP server
|
||||
- API server
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: mode
|
||||
attributes:
|
||||
label: Mode used
|
||||
options:
|
||||
- TUI (default)
|
||||
- CLI (--cli flag)
|
||||
- Subcommand (system, fit, search, list, info)
|
||||
- API server (serve)
|
||||
- MCP server
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: gpu-info
|
||||
attributes:
|
||||
label: GPU details (if relevant)
|
||||
description: Paste output of `nvidia-smi`, `rocm-smi`, or `system_profiler SPDisplaysDataType` if the bug is GPU-related.
|
||||
render: text
|
||||
|
||||
- type: textarea
|
||||
id: additional
|
||||
attributes:
|
||||
label: Additional context
|
||||
description: Any other context, screenshots, or terminal output that might help.
|
||||
@@ -0,0 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Discussions
|
||||
url: https://github.com/AlexsJones/llmfit/discussions
|
||||
about: Ask questions, share ideas, or discuss llmfit usage
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Documentation Improvement
|
||||
description: Report unclear, missing, or incorrect documentation
|
||||
title: "[Docs]: "
|
||||
labels: ["documentation"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Help us improve the llmfit documentation. Report anything that's unclear, outdated, or missing.
|
||||
|
||||
- type: dropdown
|
||||
id: doc-type
|
||||
attributes:
|
||||
label: Documentation area
|
||||
description: Which documentation needs improvement?
|
||||
options:
|
||||
- README.md (installation, usage, key bindings)
|
||||
- AGENTS.md (architecture, conventions)
|
||||
- CONTRIBUTING.md (contribution guide)
|
||||
- MODELS.md (model database info)
|
||||
- API.md (API documentation)
|
||||
- CHANGELOG.md
|
||||
- Code comments / inline docs
|
||||
- CLI help text (--help output)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: current
|
||||
attributes:
|
||||
label: Current state
|
||||
description: What is currently documented (or missing)? Link to the specific section if possible.
|
||||
placeholder: "e.g., The README doesn't explain how hardware simulation works with multi-GPU setups"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: improvement
|
||||
attributes:
|
||||
label: Suggested improvement
|
||||
description: What should the documentation say instead? What's missing?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: willingness
|
||||
attributes:
|
||||
label: Would you be willing to submit a PR for this?
|
||||
options:
|
||||
- "Yes"
|
||||
- "No, just reporting"
|
||||
validations:
|
||||
required: true
|
||||
@@ -0,0 +1,74 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or enhancement for llmfit
|
||||
title: "[Feature]: "
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
We'd love to hear your ideas for improving llmfit. Please describe your feature request below.
|
||||
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem or motivation
|
||||
description: What problem does this feature solve? Is there a workflow that's currently difficult or missing?
|
||||
placeholder: "e.g., I can't easily compare models side-by-side for my specific use case"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed solution
|
||||
description: Describe the feature you'd like to see. Be as specific as possible.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives considered
|
||||
description: Are there alternative approaches or workarounds you've considered?
|
||||
|
||||
- type: dropdown
|
||||
id: area
|
||||
attributes:
|
||||
label: Feature area
|
||||
description: Which part of llmfit would this feature affect?
|
||||
options:
|
||||
- TUI (new keybinding, view, or interaction)
|
||||
- CLI (new subcommand or flag)
|
||||
- Hardware detection (new GPU vendor, new hardware type)
|
||||
- Model database (new models, new metadata fields)
|
||||
- Fit analysis (scoring algorithm, new fit dimensions)
|
||||
- Speed estimation (tok/s accuracy, new factors)
|
||||
- Provider integration (Ollama, llama.cpp, MLX, Docker, LM Studio)
|
||||
- Download manager
|
||||
- Plan mode (hardware planning)
|
||||
- Community leaderboard
|
||||
- API / MCP server
|
||||
- Desktop app (Tauri)
|
||||
- Web interface
|
||||
- Python bindings
|
||||
- Cross-platform support
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: willingness
|
||||
attributes:
|
||||
label: Would you be willing to contribute this?
|
||||
options:
|
||||
- "Yes, I'd like to submit a PR"
|
||||
- "I could help with guidance"
|
||||
- "No, just suggesting"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: additional
|
||||
attributes:
|
||||
label: Additional context
|
||||
description: Any mockups, screenshots, links, or references that help explain the feature.
|
||||
@@ -0,0 +1,16 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: cargo
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown: # applies only to version-updates (not security-updates)
|
||||
default-days: 10
|
||||
semver-minor-days: 14 # wait 14 days before applying minor updates
|
||||
semver-major-days: 28
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 10
|
||||
@@ -0,0 +1,291 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main # Run after merging into the "main" target
|
||||
paths: # Run only if files relevant to CI have changed (i.e. not install.sh, scripts or .github/workflows)
|
||||
- 'llmfit-*/**'
|
||||
- 'Cargo.{lock,toml}'
|
||||
pull_request:
|
||||
branches:
|
||||
- main # Run in PRs targeting the "main" branch
|
||||
paths:
|
||||
- 'llmfit-*/**'
|
||||
- 'Cargo.{lock,toml}'
|
||||
types: # Avoid low-impact events like "edited" or "labeled"
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
rust-changed: ${{ steps.filter.outputs.rust }}
|
||||
python-changed: ${{ steps.filter.outputs.python }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Filter changed paths
|
||||
id: filter
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
with:
|
||||
filters: |
|
||||
rust:
|
||||
- 'llmfit-!(python)/**'
|
||||
- 'Cargo.{lock,toml}'
|
||||
python:
|
||||
- 'llmfit-python/**'
|
||||
- 'Cargo.toml'
|
||||
|
||||
test:
|
||||
name: Test Suite
|
||||
needs: changes
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
rust: [stable]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: ${{ matrix.rust }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6.4.0
|
||||
with:
|
||||
node-version: lts/*
|
||||
cache: npm
|
||||
cache-dependency-path: llmfit-web/package-lock.json
|
||||
|
||||
- name: Build web dashboard assets
|
||||
run: |
|
||||
cd llmfit-web
|
||||
npm ci
|
||||
npx vite build
|
||||
|
||||
- name: Run web unit tests
|
||||
run: |
|
||||
cd llmfit-web
|
||||
npx vitest run
|
||||
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/registry
|
||||
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-registry-
|
||||
|
||||
- name: Cache cargo index
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/git
|
||||
key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-index-
|
||||
|
||||
- name: Cache target directory
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: target
|
||||
key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-target-
|
||||
|
||||
- name: Run tests
|
||||
if: needs.changes.outputs.rust-changed == 'true'
|
||||
run: cargo test --verbose
|
||||
|
||||
# If `cargo test` already ran, most work can be reused from that step.
|
||||
- name: Build debug binary
|
||||
run: cargo build
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v8.2.0
|
||||
|
||||
# Tests integration of Rust code with hatch_build.py and key Python tests when
|
||||
# the full Python test suite is not required. Redundant if Python tests below run.
|
||||
- name: Run Rust/Python integration tests
|
||||
if: needs.changes.outputs.python-changed == 'false'
|
||||
run: uv run pytest -vv -m rust_integration
|
||||
working-directory: llmfit-python
|
||||
|
||||
- name: Run Python tests
|
||||
if: needs.changes.outputs.python-changed == 'true'
|
||||
run: uv run pytest -vv
|
||||
working-directory: llmfit-python
|
||||
|
||||
fmt:
|
||||
name: Rustfmt
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust-changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust-changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6.4.0
|
||||
with:
|
||||
node-version: lts/*
|
||||
cache: npm
|
||||
cache-dependency-path: llmfit-web/package-lock.json
|
||||
|
||||
- name: Build web dashboard assets
|
||||
run: |
|
||||
cd llmfit-web
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/registry
|
||||
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-registry-
|
||||
|
||||
- name: Cache cargo index
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/git
|
||||
key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-index-
|
||||
|
||||
- name: Cache target directory
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: target
|
||||
key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-target-
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy --all-targets --all-features
|
||||
|
||||
check:
|
||||
name: Cargo Check
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust-changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6.4.0
|
||||
with:
|
||||
node-version: lts/*
|
||||
cache: npm
|
||||
cache-dependency-path: llmfit-web/package-lock.json
|
||||
|
||||
- name: Build web dashboard assets
|
||||
run: |
|
||||
cd llmfit-web
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/registry
|
||||
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-registry-
|
||||
|
||||
- name: Cache cargo index
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cargo/git
|
||||
key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-index-
|
||||
|
||||
- name: Cache target directory
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: target
|
||||
key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-target-
|
||||
|
||||
- name: Run cargo check
|
||||
run: cargo check --all-targets --all-features
|
||||
|
||||
check-python:
|
||||
name: Python Checks
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python-changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v8.2.0
|
||||
|
||||
- name: Ruff lint
|
||||
run: uv run --only-group=dev ruff check .
|
||||
working-directory: llmfit-python
|
||||
|
||||
- name: Ruff format check
|
||||
run: uv run --only-group=dev ruff format --check .
|
||||
working-directory: llmfit-python
|
||||
|
||||
- name: Type check
|
||||
run: uv run --only-group=dev ty check .
|
||||
working-directory: llmfit-python
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Community Benchmarks
|
||||
|
||||
# Validates benchmark submissions contributed via `llmfit bench --share`
|
||||
# (PRs adding files under llmfit-core/data/community/): JSON schema
|
||||
# conformance, path/naming conventions, and cross-field sanity checks.
|
||||
|
||||
'on':
|
||||
pull_request:
|
||||
paths:
|
||||
- 'llmfit-core/data/community/**'
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'llmfit-core/data/community/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate submissions
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install jsonschema
|
||||
run: pip install jsonschema
|
||||
|
||||
# Validates the whole directory, not just changed files: repo-wide
|
||||
# integrity is the invariant, and the directory stays small.
|
||||
- name: Validate community submissions
|
||||
run: python3 scripts/validate_community_benchmarks.py
|
||||
@@ -0,0 +1,59 @@
|
||||
name: Docker Build and Push
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
- "!v*-mac"
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.1.0
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Release Desktop (macOS)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*-mac"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
build-desktop:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: aarch64-apple-darwin
|
||||
os: macos-latest
|
||||
- target: x86_64-apple-darwin
|
||||
os: macos-latest
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install Tauri CLI
|
||||
run: cargo install tauri-cli --version "^2"
|
||||
|
||||
- name: Build Tauri app bundle
|
||||
run: cargo tauri build --target ${{ matrix.target }} --bundles app
|
||||
working-directory: llmfit-desktop
|
||||
|
||||
- name: Package .app bundle
|
||||
shell: bash
|
||||
run: |
|
||||
TAG="${GITHUB_REF_NAME}"
|
||||
# Search both possible target locations
|
||||
for BASE in "target" "llmfit-desktop/target"; do
|
||||
APP=$(find "${BASE}/${{ matrix.target }}/release/bundle" -name '*.app' -maxdepth 3 2>/dev/null | head -1)
|
||||
[ -n "$APP" ] && break
|
||||
done
|
||||
|
||||
if [ -z "$APP" ]; then
|
||||
echo "::error::No .app bundle found"
|
||||
find target/ llmfit-desktop/target/ -type d -name 'bundle' 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found app bundle: $APP"
|
||||
DEST="llmfit-desktop-${TAG}-${{ matrix.target }}.app.tar.gz"
|
||||
cd "$(dirname "$APP")"
|
||||
tar czf "/tmp/${DEST}" "$(basename "$APP")"
|
||||
echo "DESKTOP_ASSET=${DEST}" >> "$GITHUB_ENV"
|
||||
echo "DESKTOP_ASSET_PATH=/tmp/${DEST}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ env.DESKTOP_ASSET }}
|
||||
path: ${{ env.DESKTOP_ASSET_PATH }}
|
||||
|
||||
release:
|
||||
needs: build-desktop
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
||||
with:
|
||||
generate_release_notes: true
|
||||
files: artifacts/**/*.tar.gz
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Release Please
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
release-please:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: googleapis/release-please-action@v5.0.0
|
||||
id: release
|
||||
with:
|
||||
token: ${{ secrets.RELEASE_TOKEN }}
|
||||
config-file: release-please-config.json
|
||||
manifest-file: .release-please-manifest.json
|
||||
|
||||
- uses: actions/checkout@v7.0.0
|
||||
if: steps.release.outputs.pr
|
||||
with:
|
||||
ref: ${{ fromJson(steps.release.outputs.pr).headBranchName }}
|
||||
token: ${{ secrets.RELEASE_TOKEN }}
|
||||
|
||||
- name: Update Cargo.lock
|
||||
if: steps.release.outputs.pr
|
||||
run: cargo update --workspace
|
||||
|
||||
- name: Commit updated Cargo.lock
|
||||
if: steps.release.outputs.pr
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add Cargo.lock
|
||||
git diff --cached --quiet || git commit -m "chore: update Cargo.lock"
|
||||
git push
|
||||
@@ -0,0 +1,433 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
# Fires when release-please (or a maintainer) publishes a GitHub Release.
|
||||
# Using the `release` event instead of `push: tags` is necessary because
|
||||
# tags created by the release-please workflow use GITHUB_TOKEN, which by
|
||||
# design does not trigger downstream workflows — that caused v0.9.4 to
|
||||
# ship with no binary assets (see issue #424).
|
||||
release:
|
||||
types: [published]
|
||||
# Manual backfill: run against a specific existing tag (e.g. v0.9.4) to
|
||||
# upload assets to an already-published release.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing tag to build and attach assets to (e.g. v0.9.4)"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
BINARY: llmfit
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# Linux x86_64 (static musl)
|
||||
- target: x86_64-unknown-linux-musl
|
||||
os: ubuntu-latest
|
||||
use-cross: true
|
||||
|
||||
# Linux ARM64 (static musl)
|
||||
- target: aarch64-unknown-linux-musl
|
||||
os: ubuntu-latest
|
||||
use-cross: true
|
||||
|
||||
# Linux x86_64 (glibc)
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
os: ubuntu-latest
|
||||
use-cross: false
|
||||
|
||||
# Linux ARM64 (glibc)
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
os: ubuntu-latest
|
||||
use-cross: true
|
||||
|
||||
# Linux RISCV64 (glibc)
|
||||
- target: riscv64gc-unknown-linux-gnu
|
||||
os: ubuntu-latest
|
||||
use-cross: true
|
||||
|
||||
# macOS Intel (cross-compiled from ARM64 runner)
|
||||
- target: x86_64-apple-darwin
|
||||
os: macos-latest
|
||||
use-cross: false
|
||||
|
||||
# macOS Apple Silicon
|
||||
- target: aarch64-apple-darwin
|
||||
os: macos-latest
|
||||
use-cross: false
|
||||
|
||||
# Windows x86_64
|
||||
- target: x86_64-pc-windows-msvc
|
||||
os: windows-latest
|
||||
use-cross: false
|
||||
|
||||
# Windows ARM64
|
||||
- target: aarch64-pc-windows-msvc
|
||||
os: windows-latest
|
||||
use-cross: false
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Build web dashboard assets
|
||||
run: |
|
||||
cd llmfit-web
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use-cross
|
||||
run: cargo install cross --version 0.2.5
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.use-cross }}" = "true" ]; then
|
||||
cross build --release --target ${{ matrix.target }}
|
||||
else
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
fi
|
||||
|
||||
- name: Package
|
||||
shell: bash
|
||||
run: |
|
||||
TAG="${{ inputs.tag || github.ref_name }}"
|
||||
ASSET_NAME="${BINARY}-${TAG}-${{ matrix.target }}"
|
||||
STAGING="${RUNNER_TEMP}/${ASSET_NAME}"
|
||||
mkdir -p "${STAGING}"
|
||||
|
||||
if [[ "${{ matrix.target }}" == *"windows"* ]]; then
|
||||
EXE_EXT=".exe"
|
||||
ARCHIVE_EXT=".zip"
|
||||
COMPRESS_CMD="7z a ${ASSET_NAME}${ARCHIVE_EXT} ${ASSET_NAME}"
|
||||
else
|
||||
EXE_EXT=""
|
||||
ARCHIVE_EXT=".tar.gz"
|
||||
COMPRESS_CMD="tar czf ${ASSET_NAME}${ARCHIVE_EXT} ${ASSET_NAME}"
|
||||
fi
|
||||
|
||||
cp "target/${{ matrix.target }}/release/${BINARY}${EXE_EXT}" "${STAGING}/"
|
||||
cp README.md LICENSE "${STAGING}/" 2>/dev/null || true
|
||||
|
||||
cd "${RUNNER_TEMP}"
|
||||
$COMPRESS_CMD
|
||||
|
||||
# Generate per-asset SHA256 checksum file (consistent format for both tools)
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "${ASSET_NAME}${ARCHIVE_EXT}" | awk '{print $1 " " $2}' > "${ASSET_NAME}${ARCHIVE_EXT}.sha256"
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
shasum -a 256 "${ASSET_NAME}${ARCHIVE_EXT}" | awk '{print $1 " " $2}' > "${ASSET_NAME}${ARCHIVE_EXT}.sha256"
|
||||
fi
|
||||
|
||||
echo "ASSET=${ASSET_NAME}${ARCHIVE_EXT}" >> "$GITHUB_ENV"
|
||||
echo "ASSET_PATH=${RUNNER_TEMP}/${ASSET_NAME}${ARCHIVE_EXT}" >> "$GITHUB_ENV"
|
||||
echo "CHECKSUM_PATH=${RUNNER_TEMP}/${ASSET_NAME}${ARCHIVE_EXT}.sha256" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ env.ASSET }}
|
||||
path: |
|
||||
${{ env.ASSET_PATH }}
|
||||
${{ env.CHECKSUM_PATH }}
|
||||
|
||||
# Sign Windows executables via SignPath (Authenticode).
|
||||
# Uses the test-signing policy initially; switch to release-signing
|
||||
# after SignPath reviews the setup and provisions the production cert.
|
||||
sign-windows:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
target:
|
||||
- x86_64-pc-windows-msvc
|
||||
- aarch64-pc-windows-msvc
|
||||
steps:
|
||||
- name: Determine asset name
|
||||
id: asset-name
|
||||
shell: bash
|
||||
run: |
|
||||
TAG="${{ inputs.tag || github.ref_name }}"
|
||||
ASSET="llmfit-${TAG}-${{ matrix.target }}"
|
||||
echo "zip=${ASSET}.zip" >> "$GITHUB_OUTPUT"
|
||||
echo "dir=${ASSET}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download unsigned artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: ${{ steps.asset-name.outputs.zip }}
|
||||
path: unsigned
|
||||
|
||||
- name: Extract exe from zip
|
||||
shell: bash
|
||||
run: |
|
||||
cd unsigned
|
||||
unzip "${{ steps.asset-name.outputs.zip }}"
|
||||
# Move the exe to a clean directory for signing
|
||||
mkdir -p ../to-sign
|
||||
cp "${{ steps.asset-name.outputs.dir }}/llmfit.exe" ../to-sign/
|
||||
|
||||
- name: Upload exe for SignPath
|
||||
id: upload-for-signing
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ steps.asset-name.outputs.zip }}-unsigned
|
||||
path: to-sign/llmfit.exe
|
||||
|
||||
- name: Submit signing request to SignPath
|
||||
uses: signpath/github-action-submit-signing-request@v2
|
||||
with:
|
||||
api-token: ${{ secrets.SIGNPATH_API_TOKEN }}
|
||||
organization-id: ${{ secrets.SIGNPATH_ORGANIZATION_ID }}
|
||||
project-slug: ${{ secrets.SIGNPATH_PROJECT_SLUG }}
|
||||
signing-policy-slug: ${{ secrets.SIGNPATH_SIGNING_POLICY_SLUG }}
|
||||
artifact-configuration-slug: ${{ secrets.SIGNPATH_ARTIFACT_CONFIGURATION_SLUG }}
|
||||
github-artifact-id: ${{ steps.upload-for-signing.outputs.artifact-id }}
|
||||
wait-for-completion: true
|
||||
output-artifact-directory: signed
|
||||
|
||||
- name: Repackage signed exe into zip
|
||||
shell: bash
|
||||
run: |
|
||||
# Replace the unsigned exe with the signed one
|
||||
cp signed/llmfit.exe "unsigned/${{ steps.asset-name.outputs.dir }}/llmfit.exe"
|
||||
cd unsigned
|
||||
zip -r "../${{ steps.asset-name.outputs.zip }}" "${{ steps.asset-name.outputs.dir }}"
|
||||
|
||||
- name: Replace unsigned artifact with signed one
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ steps.asset-name.outputs.zip }}
|
||||
path: ${{ steps.asset-name.outputs.zip }}
|
||||
overwrite: true
|
||||
|
||||
- name: Regenerate checksum for signed zip
|
||||
shell: bash
|
||||
run: |
|
||||
sha256sum "${{ steps.asset-name.outputs.zip }}" | awk '{print $1 " " $2}' > "${{ steps.asset-name.outputs.zip }}.sha256"
|
||||
|
||||
- name: Upload signed checksum
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ steps.asset-name.outputs.zip }}.sha256
|
||||
path: ${{ steps.asset-name.outputs.zip }}.sha256
|
||||
overwrite: true
|
||||
|
||||
release:
|
||||
needs: [build, sign-windows]
|
||||
# Run even if sign-windows is skipped (e.g. secrets not yet configured)
|
||||
if: always() && needs.build.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
||||
with:
|
||||
tag_name: ${{ inputs.tag || github.ref_name }}
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
artifacts/**/*.tar.gz
|
||||
artifacts/**/*.zip
|
||||
artifacts/**/*.sha256
|
||||
|
||||
publish-python:
|
||||
needs: release
|
||||
# Run when release succeeds, even if upstream sign-windows was skipped or failed
|
||||
if: always() && needs.release.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/p/llmfit/
|
||||
permissions:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@v8.2.0
|
||||
|
||||
- name: Build Python wheels
|
||||
shell: bash
|
||||
env:
|
||||
TAG: ${{ inputs.tag || github.ref_name }}
|
||||
run: |
|
||||
# Each entry: python_platform_tag:rust_target:binary_name:archive_ext
|
||||
# Must stay in sync with TARGET_CONFIGS in llmfit-python/hatch_build.py.
|
||||
TARGETS=(
|
||||
"manylinux_2_17_x86_64:x86_64-unknown-linux-gnu:llmfit:tar.gz"
|
||||
"manylinux_2_17_aarch64:aarch64-unknown-linux-gnu:llmfit:tar.gz"
|
||||
"manylinux_2_39_riscv64:riscv64gc-unknown-linux-gnu:llmfit:tar.gz"
|
||||
"musllinux_1_2_x86_64:x86_64-unknown-linux-musl:llmfit:tar.gz"
|
||||
"musllinux_1_2_aarch64:aarch64-unknown-linux-musl:llmfit:tar.gz"
|
||||
"macosx_10_12_x86_64:x86_64-apple-darwin:llmfit:tar.gz"
|
||||
"macosx_11_0_arm64:aarch64-apple-darwin:llmfit:tar.gz"
|
||||
"win_amd64:x86_64-pc-windows-msvc:llmfit.exe:zip"
|
||||
"win_arm64:aarch64-pc-windows-msvc:llmfit.exe:zip"
|
||||
)
|
||||
BASE_URL="https://github.com/AlexsJones/llmfit/releases/download/${TAG}"
|
||||
mkdir -p dist/
|
||||
for entry in "${TARGETS[@]}"; do
|
||||
IFS=: read -r py_tag rust_target binary_name archive_ext <<< "$entry"
|
||||
ASSET="llmfit-${TAG}-${rust_target}"
|
||||
ARCHIVE="${ASSET}.${archive_ext}"
|
||||
EXPECTED=$(curl -fsSL "${BASE_URL}/${ARCHIVE}.sha256" | awk '{print $1}')
|
||||
ACTUAL=$(curl -fsSL "${BASE_URL}/${ARCHIVE}" | tee "${ARCHIVE}" | sha256sum | awk '{print $1}')
|
||||
if [ "${ACTUAL}" != "${EXPECTED}" ]; then
|
||||
echo "Checksum mismatch for ${ARCHIVE}: expected ${EXPECTED}, got ${ACTUAL}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Checksum OK for ${ARCHIVE}"
|
||||
if [ "${archive_ext}" = "tar.gz" ]; then
|
||||
tar -xzf "${ARCHIVE}" "${ASSET}/${binary_name}"
|
||||
else
|
||||
unzip "${ARCHIVE}" "${ASSET}/${binary_name}"
|
||||
fi
|
||||
dest="target/${rust_target}/release"
|
||||
mkdir -p "${dest}"
|
||||
cp "${ASSET}/${binary_name}" "${dest}/${binary_name}"
|
||||
chmod +x "${dest}/${binary_name}"
|
||||
rm -f "${ARCHIVE}"
|
||||
rm -rf "${ASSET}"
|
||||
LLMFIT_PYTHON_PLATFORM_TAG="${py_tag}" uv build --wheel --out-dir=dist/ llmfit-python/
|
||||
done
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
publish-crate:
|
||||
needs: release
|
||||
if: always() && needs.release.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# Pass the registry token via env rather than --token: argv is
|
||||
# visible in the process listing on the runner, and GitHub secret
|
||||
# masking only covers log output. cargo reads CARGO_REGISTRY_TOKEN
|
||||
# automatically.
|
||||
- name: Publish llmfit-core to crates.io
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
run: cargo publish -p llmfit-core
|
||||
|
||||
- name: Wait for crates.io index update
|
||||
run: sleep 30
|
||||
|
||||
- name: Publish llmfit to crates.io
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
run: cargo publish -p llmfit
|
||||
|
||||
update-homebrew:
|
||||
needs: release
|
||||
if: always() && needs.release.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout tap
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
repository: AlexsJones/homebrew-llmfit
|
||||
token: ${{ secrets.HOMEBREW_TAP_TOKEN }}
|
||||
path: homebrew-tap
|
||||
|
||||
- name: Download release assets and update formula
|
||||
env:
|
||||
TAG: ${{ inputs.tag || github.ref_name }}
|
||||
run: |
|
||||
VERSION="${TAG#v}"
|
||||
|
||||
# Download the four tarballs and compute SHA256
|
||||
declare -A SHAS
|
||||
for target in aarch64-apple-darwin x86_64-apple-darwin aarch64-unknown-linux-musl x86_64-unknown-linux-musl; do
|
||||
URL="https://github.com/AlexsJones/llmfit/releases/download/${TAG}/llmfit-${TAG}-${target}.tar.gz"
|
||||
echo "Downloading ${URL}..."
|
||||
SHA=$(curl -fsSL "$URL" | shasum -a 256 | awk '{print $1}')
|
||||
SHAS[$target]="$SHA"
|
||||
echo "${target}: ${SHA}"
|
||||
done
|
||||
|
||||
# Generate the formula
|
||||
cat > homebrew-tap/Formula/llmfit.rb << RUBY
|
||||
class Llmfit < Formula
|
||||
desc "Terminal tool that right-sizes LLM models to your system hardware"
|
||||
homepage "https://github.com/AlexsJones/llmfit"
|
||||
version "${VERSION}"
|
||||
license "MIT"
|
||||
|
||||
on_macos do
|
||||
if Hardware::CPU.arm?
|
||||
url "https://github.com/AlexsJones/llmfit/releases/download/v#{version}/llmfit-v#{version}-aarch64-apple-darwin.tar.gz"
|
||||
sha256 "${SHAS[aarch64-apple-darwin]}"
|
||||
else
|
||||
url "https://github.com/AlexsJones/llmfit/releases/download/v#{version}/llmfit-v#{version}-x86_64-apple-darwin.tar.gz"
|
||||
sha256 "${SHAS[x86_64-apple-darwin]}"
|
||||
end
|
||||
end
|
||||
|
||||
on_linux do
|
||||
if Hardware::CPU.arm?
|
||||
url "https://github.com/AlexsJones/llmfit/releases/download/v#{version}/llmfit-v#{version}-aarch64-unknown-linux-musl.tar.gz"
|
||||
sha256 "${SHAS[aarch64-unknown-linux-musl]}"
|
||||
else
|
||||
url "https://github.com/AlexsJones/llmfit/releases/download/v#{version}/llmfit-v#{version}-x86_64-unknown-linux-musl.tar.gz"
|
||||
sha256 "${SHAS[x86_64-unknown-linux-musl]}"
|
||||
end
|
||||
end
|
||||
|
||||
def install
|
||||
bin.install "llmfit"
|
||||
end
|
||||
|
||||
test do
|
||||
assert_match "llmfit", shell_output("#{bin}/llmfit --help")
|
||||
end
|
||||
end
|
||||
RUBY
|
||||
|
||||
# Fix heredoc indentation
|
||||
sed -i 's/^ //' homebrew-tap/Formula/llmfit.rb
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
cd homebrew-tap
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add Formula/llmfit.rb
|
||||
git commit -m "Update llmfit to ${TAG}"
|
||||
git push
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Weekly Model Update
|
||||
|
||||
'on':
|
||||
schedule:
|
||||
- cron: '0 2 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: weekly-model-update
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
update-models:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
# HF_TOKEN raises HuggingFace API rate limits and unlocks gated-model
|
||||
# metadata (the scraper warns-and-skips gated repos without it).
|
||||
- name: Refresh model database
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: python3 scripts/scrape_hf_models.py -n 5000
|
||||
|
||||
# Refresh the community leaderboard cache alongside the catalog. The
|
||||
# measured tok/s rows also feed the estimate-calibration test below,
|
||||
# so this doubles as a weekly accuracy audit of estimate_tps.
|
||||
- name: Refresh benchmark cache
|
||||
env:
|
||||
LOCALMAXXING_API_KEY: ${{ secrets.LOCALMAXXING_API_KEY }}
|
||||
run: python3 scripts/scrape_benchmarks.py --limit 200
|
||||
|
||||
- name: Validate generated JSON
|
||||
run: |
|
||||
python3 -m json.tool llmfit-core/data/hf_models.json > /dev/null
|
||||
python3 -m json.tool llmfit-core/data/benchmark_cache.json > /dev/null
|
||||
|
||||
- name: Set up Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
- name: Validate against JSON Schema
|
||||
run: cargo test -p llmfit-core hf_models_match_schema
|
||||
|
||||
# Weekly accuracy audit: replay the refreshed measurements through
|
||||
# estimate_tps; fails on systematic estimator bias (see fit.rs).
|
||||
- name: Calibrate estimates against measured benchmarks
|
||||
run: cargo test -p llmfit-core test_estimate_tps_calibration -- --nocapture
|
||||
|
||||
- name: Check for changes
|
||||
id: changes
|
||||
run: |
|
||||
if git diff --quiet; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create or update pull request
|
||||
if: steps.changes.outputs.changed == 'true'
|
||||
uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
commit-message: "chore(models): Weekly model and benchmark data refresh"
|
||||
title: "[Automated] Weekly model and benchmark data refresh"
|
||||
body: |
|
||||
Automated weekly refresh of model metadata (`scripts/scrape_hf_models.py`) and the localmaxxing benchmark cache (`scripts/scrape_benchmarks.py`).
|
||||
|
||||
**Note:** CI checks may not automatically trigger with the default `github.token`. If checks don't run, please close and reopen the PR or manually trigger them.
|
||||
branch: automated/weekly-model-update
|
||||
delete-branch: true
|
||||
labels: |
|
||||
automated
|
||||
data-update
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
.DS_Store
|
||||
/target
|
||||
llmfit-desktop/gen/schemas/*
|
||||
data/gguf_sources_cache.json
|
||||
data/hf_models.json.backup.*
|
||||
llmfit-web/node_modules/
|
||||
llmfit-web/dist/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
dist/
|
||||
.venv/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
".": "1.1.2"
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
# AGENTS.md
|
||||
|
||||
Instructions for AI agents contributing to this codebase.
|
||||
|
||||
---
|
||||
|
||||
## Project overview
|
||||
|
||||
`llmfit` is a Rust CLI/TUI tool that matches LLM models against local system hardware (RAM, CPU, GPU). It detects system specs, loads a model database from embedded JSON, scores each model's fit, and presents results in an interactive terminal UI or classic table output.
|
||||
|
||||
## Language and toolchain
|
||||
|
||||
- Rust, edition 2024.
|
||||
- Build with `cargo build`. Run with `cargo run`.
|
||||
- No nightly features required. Stable toolchain only.
|
||||
- Minimum supported Rust version: whatever edition 2024 requires (1.85+).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
main.rs Entrypoint. Parses CLI args via clap. Launches TUI by default,
|
||||
falls back to CLI subcommands (system, list, fit, search, info)
|
||||
or --cli flag for classic table output.
|
||||
|
||||
hardware.rs SystemSpecs::detect() reads RAM/CPU via sysinfo crate.
|
||||
detect_gpu() shells out to nvidia-smi / rocm-smi, and
|
||||
detects Apple Silicon via system_profiler.
|
||||
On unified memory (Apple Silicon), VRAM = system RAM.
|
||||
No async. No unsafe.
|
||||
|
||||
models.rs LlmModel struct. ModelDatabase loads from llmfit-core/data/hf_models.json
|
||||
embedded via include_str!() at compile time. No runtime file I/O.
|
||||
|
||||
fit.rs FitLevel enum (Perfect, Good, Marginal, TooTight).
|
||||
RunMode enum (Gpu, CpuOffload, CpuOnly).
|
||||
ModelFit::analyze() compares a model against SystemSpecs,
|
||||
selecting the best available execution path (GPU > CPU offload > CPU).
|
||||
rank_models_by_fit() sorts by fit level, then run mode, then utilization.
|
||||
|
||||
display.rs CLI-mode table rendering using the tabled crate.
|
||||
Only used when --cli flag or subcommands are invoked.
|
||||
|
||||
tui_app.rs TUI application state. Holds all models, filters (search text,
|
||||
provider toggles, fit filter), selection index.
|
||||
All filtering logic is here -- apply_filters() recomputes
|
||||
filtered_fits indices whenever inputs change.
|
||||
|
||||
tui_ui.rs Rendering with ratatui. Four layout regions: system bar,
|
||||
search/filter bar, model table (or detail pane), status bar.
|
||||
Stateless rendering -- reads from App, writes to Frame.
|
||||
|
||||
tui_events.rs Keyboard event handling with crossterm. Two modes: Normal
|
||||
(navigation, filter toggling, quit) and Search (text input).
|
||||
```
|
||||
|
||||
## Data flow
|
||||
|
||||
1. `App::new()` calls `SystemSpecs::detect()` and `ModelDatabase::new()`.
|
||||
2. Every model is analyzed into a `ModelFit` via `ModelFit::analyze()`.
|
||||
3. Results are sorted by `rank_models_by_fit()`.
|
||||
4. `apply_filters()` produces `filtered_fits: Vec<usize>` (indices into `all_fits`).
|
||||
5. The TUI render loop reads `App` state and draws via `tui_ui::draw()`.
|
||||
6. `tui_events::handle_events()` mutates `App` state, triggering re-render.
|
||||
|
||||
## Model database
|
||||
|
||||
- Source: `llmfit-core/data/hf_models.json` (33 models).
|
||||
- Generated by `scripts/scrape_hf_models.py` (Python, stdlib only, no pip deps).
|
||||
- Embedded at compile time via `include_str!("../data/hf_models.json")`.
|
||||
- Schema per entry: name, provider, parameter_count, min_ram_gb, recommended_ram_gb, min_vram_gb, quantization, context_length, use_case.
|
||||
- `min_vram_gb` is VRAM needed for GPU inference. `min_ram_gb` is system RAM needed for CPU inference. Both are derived from the same parameter count.
|
||||
- RAM formula: `params * 0.5 bytes (Q4_K_M) / 1024^3 * 1.2 overhead`.
|
||||
- VRAM formula: `params * 0.5 bytes (Q4_K_M) / 1024^3 * 1.1 activation overhead`.
|
||||
- Recommended RAM: `model_size * 2.0`.
|
||||
|
||||
Do not manually edit `hf_models.json`. Regenerate it by running the scraper:
|
||||
|
||||
```sh
|
||||
python3 scripts/scrape_hf_models.py
|
||||
```
|
||||
|
||||
The scraper has hardcoded fallback entries for gated models that require authentication.
|
||||
|
||||
## Conventions
|
||||
|
||||
- No `unsafe` code.
|
||||
- No `.unwrap()` on user-facing paths. Use proper error handling or `expect()` with a descriptive message for internal invariants only.
|
||||
- Fit levels are ordered: Perfect > Good > Marginal > TooTight. Do not add levels without updating `rank_models_by_fit()` sort logic.
|
||||
- Fit is VRAM-first. GPU inference with sufficient VRAM is the ideal path. CPU inference via system RAM is a fallback. The `RunMode` enum tracks which memory pool is being used (Gpu, CpuOffload, CpuOnly).
|
||||
- `min_vram_gb` is the VRAM needed to load model weights on GPU. `min_ram_gb` is the system RAM needed for CPU-only inference (same weights, loaded into RAM instead). They represent the same workload on different hardware paths.
|
||||
- On Apple Silicon (unified memory), VRAM = system RAM. The `CpuOffload` path is skipped because there is no separate RAM pool to spill to. `SystemSpecs::unified_memory` tracks this.
|
||||
- TUI rendering is stateless. `tui_ui::draw()` must not mutate `App`. Pass `&mut App` only for `TableState` widget requirements -- do not use it to change application state.
|
||||
- Event handling in `tui_events.rs` is the sole place that mutates `App` in the TUI loop.
|
||||
- Keep `display.rs` and `tui_*.rs` independent. The CLI path must work without initializing any TUI state.
|
||||
|
||||
## Adding a new model to the database
|
||||
|
||||
1. Add the model's HuggingFace repo ID to `TARGET_MODELS` in `scripts/scrape_hf_models.py`.
|
||||
2. If the model is gated (requires HF auth), add a fallback entry to the `FALLBACK` dict in the same script.
|
||||
3. Run `python3 scripts/scrape_hf_models.py`.
|
||||
4. Verify the output in `llmfit-core/data/hf_models.json`.
|
||||
5. Run `cargo build` to verify compilation.
|
||||
|
||||
## Adding a new filter
|
||||
|
||||
1. Add the filter state to `App` in `tui_app.rs`.
|
||||
2. Add filtering logic inside `apply_filters()`.
|
||||
3. Add the keybinding in `tui_events.rs` (Normal mode handler).
|
||||
4. Add the UI widget in `tui_ui.rs` (`draw_search_and_filters()` function).
|
||||
5. Update the status bar help text in `draw_status_bar()`.
|
||||
|
||||
## Adding a new CLI subcommand
|
||||
|
||||
1. Add a variant to the `Commands` enum in `main.rs`.
|
||||
2. Add the match arm in the `main()` function's command dispatch.
|
||||
3. Use `display.rs` functions for output, or add new ones as needed.
|
||||
|
||||
## Testing
|
||||
|
||||
There are no tests yet. When adding tests:
|
||||
|
||||
- Unit tests for `fit.rs` logic (given known SystemSpecs and LlmModel values, assert correct FitLevel).
|
||||
- Unit tests for `models.rs` (verify JSON parsing, search matching).
|
||||
- Integration tests for CLI subcommands via `assert_cmd` crate.
|
||||
- TUI is difficult to unit test. Keep rendering stateless and test the state mutations in `tui_app.rs` directly.
|
||||
|
||||
## Dependencies policy
|
||||
|
||||
- Prefer crates that are well-maintained and have minimal transitive dependencies.
|
||||
- `sysinfo` is the system detection crate. Do not replace it with raw platform calls.
|
||||
- `ratatui` + `crossterm` is the TUI stack. Do not mix in `termion` or `ncurses`.
|
||||
- `clap` with derive feature for CLI parsing. Do not use manual arg parsing.
|
||||
- The Python scraper uses only stdlib (`urllib`, `json`). Do not add pip dependencies.
|
||||
|
||||
## Common tasks
|
||||
|
||||
```sh
|
||||
# Build
|
||||
cargo build
|
||||
|
||||
# Run TUI
|
||||
cargo run
|
||||
|
||||
# Run CLI mode
|
||||
cargo run -- --cli
|
||||
|
||||
# Run specific subcommand
|
||||
cargo run -- system
|
||||
cargo run -- fit --perfect -n 5
|
||||
cargo run -- search "llama"
|
||||
|
||||
# Refresh model database
|
||||
python3 scripts/scrape_hf_models.py && cargo build
|
||||
|
||||
# Check for compilation issues
|
||||
cargo check
|
||||
|
||||
# Format code
|
||||
cargo fmt
|
||||
|
||||
# Lint
|
||||
cargo clippy
|
||||
```
|
||||
|
||||
## Platform notes
|
||||
|
||||
- GPU detection shells out to `nvidia-smi` (NVIDIA) and `rocm-smi` (AMD). These are best-effort and fail silently if unavailable.
|
||||
- Apple Silicon detection uses `system_profiler SPDisplaysDataType`. On unified memory Macs, VRAM is reported as available system RAM (same pool).
|
||||
- `sysinfo` handles cross-platform RAM/CPU. No conditional compilation needed.
|
||||
- The TUI uses crossterm which works on Linux, macOS, and Windows terminals.
|
||||
@@ -0,0 +1,351 @@
|
||||
# llmfit REST API Guide
|
||||
|
||||
This document is for agent/client builders integrating with `llmfit serve`.
|
||||
|
||||
## Purpose
|
||||
|
||||
`llmfit serve` exposes node-local model fit analysis (same core data used by TUI/CLI) over HTTP and serves a local web dashboard.
|
||||
|
||||
Primary use case:
|
||||
- Query each node in a cluster for top runnable models.
|
||||
- Aggregate externally (scheduler/controller/UI) for placement decisions.
|
||||
|
||||
## Start the server
|
||||
|
||||
```sh
|
||||
llmfit serve --port 8787
|
||||
```
|
||||
|
||||
Global flags still apply:
|
||||
|
||||
```sh
|
||||
llmfit --memory 24G --ram 64G --cpu-cores 16 --max-context 8192 serve --port 8787
|
||||
```
|
||||
|
||||
Hardware overrides (`--memory`, `--ram`, `--cpu-cores`) are reflected in API responses, making the server report the overridden values instead of the detected hardware.
|
||||
|
||||
## Base URL
|
||||
|
||||
Default local base URL:
|
||||
|
||||
```text
|
||||
http://127.0.0.1:8787
|
||||
```
|
||||
|
||||
To expose outside localhost, pass `--host 0.0.0.0`.
|
||||
|
||||
### Unix domain socket
|
||||
|
||||
For same-host consumers that should not touch the network at all (e.g. a
|
||||
sidecar in a `hostNetwork` Kubernetes pod, where a TCP bind would land on the
|
||||
node's loopback), listen on a Unix socket instead:
|
||||
|
||||
```sh
|
||||
llmfit serve --unix-socket /run/llmfit/llmfit.sock
|
||||
```
|
||||
|
||||
The socket is created with mode `0660`; a stale socket file from a previous
|
||||
instance is replaced automatically. All HTTP endpoints are identical:
|
||||
|
||||
```sh
|
||||
curl --unix-socket /run/llmfit/llmfit.sock http://localhost/api/v1/system
|
||||
```
|
||||
|
||||
`--unix-socket` conflicts with `--host`/`--port` and is unix-platforms only.
|
||||
|
||||
If you are building from source and want the dashboard embedded in `llmfit`, build web assets first:
|
||||
|
||||
```sh
|
||||
cd llmfit-web && npm ci && npm run build
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
### `GET /`
|
||||
Web dashboard entrypoint (same-origin UI for fit exploration).
|
||||
|
||||
### `GET /health`
|
||||
Liveness probe.
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"node": {
|
||||
"name": "worker-1",
|
||||
"os": "linux"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/v1/system`
|
||||
Returns node identity + detected hardware.
|
||||
|
||||
Example response shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"node": {
|
||||
"name": "worker-1",
|
||||
"os": "linux"
|
||||
},
|
||||
"system": {
|
||||
"total_ram_gb": 62.23,
|
||||
"available_ram_gb": 41.08,
|
||||
"cpu_cores": 14,
|
||||
"cpu_name": "Intel(R) Core(TM) Ultra 7 165U",
|
||||
"has_gpu": false,
|
||||
"gpu_vram_gb": null,
|
||||
"gpu_name": null,
|
||||
"gpu_count": 0,
|
||||
"unified_memory": false,
|
||||
"backend": "CPU (x86)",
|
||||
"gpus": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/v1/models`
|
||||
Returns filtered/sorted model-fit rows for this node.
|
||||
|
||||
Envelope shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"node": { "name": "worker-1", "os": "linux" },
|
||||
"system": { "...": "..." },
|
||||
"total_models": 23,
|
||||
"returned_models": 10,
|
||||
"filters": { "...": "echo of query state" },
|
||||
"models": [
|
||||
{
|
||||
"name": "Qwen/Qwen2.5-Coder-7B-Instruct",
|
||||
"provider": "Qwen",
|
||||
"parameter_count": "7B",
|
||||
"params_b": 7.0,
|
||||
"context_length": 32768,
|
||||
"use_case": "Coding",
|
||||
"category": "Coding",
|
||||
"release_date": "2025-03-14",
|
||||
"is_moe": false,
|
||||
"fit_level": "good",
|
||||
"fit_label": "Good",
|
||||
"run_mode": "gpu",
|
||||
"run_mode_label": "GPU",
|
||||
"score": 86.5,
|
||||
"score_components": {
|
||||
"quality": 87.0,
|
||||
"speed": 81.2,
|
||||
"fit": 90.1,
|
||||
"context": 88.0
|
||||
},
|
||||
"estimated_tps": 42.5,
|
||||
"runtime": "llamacpp",
|
||||
"runtime_label": "llama.cpp",
|
||||
"best_quant": "Q5_K_M",
|
||||
"memory_required_gb": 5.8,
|
||||
"memory_available_gb": 12.0,
|
||||
"utilization_pct": 48.3,
|
||||
"notes": [],
|
||||
"gguf_sources": []
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/v1/models/top`
|
||||
Key scheduling endpoint. Same schema as `/api/v1/models`, but defaults to top 5 runnable entries.
|
||||
|
||||
Important behavior:
|
||||
- Defaults `limit=5`.
|
||||
- Excludes `too_tight` rows unless explicitly overridden (and top endpoint still keeps runnable semantics).
|
||||
|
||||
---
|
||||
|
||||
### `GET /api/v1/models/{name}`
|
||||
Path-constrained search. Equivalent to a text search scoped by `{name}`.
|
||||
|
||||
Useful for:
|
||||
- Client-side drilldown after selecting a model family.
|
||||
|
||||
## Query parameters
|
||||
|
||||
Supported on `/api/v1/models` and `/api/v1/models/top` (also `/api/v1/models/{name}`):
|
||||
|
||||
- `limit` (or alias `n`): max rows returned.
|
||||
- `perfect`: `true|false` (when `true`, only perfect fits).
|
||||
- `min_fit`: `perfect|good|marginal|too_tight`.
|
||||
- `runtime`: `any|mlx|llamacpp`.
|
||||
- `use_case`: `general|coding|reasoning|chat|multimodal|embedding`.
|
||||
- `provider`: provider substring filter.
|
||||
- `search`: free-text filter (name/provider/params/use-case/category).
|
||||
- `sort`: `score|tps|params|mem|ctx|date|use_case`.
|
||||
- `include_too_tight`: include unrunnable rows (defaults true for `/models`, false for `/models/top`).
|
||||
- `max_context`: per-request context cap used by memory estimation.
|
||||
- `force_runtime`: `mlx|llamacpp|vllm` — override automatic runtime selection during analysis (e.g. get llama.cpp recommendations on Apple Silicon instead of MLX).
|
||||
|
||||
## Error handling
|
||||
|
||||
Invalid filter values return HTTP 400:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": "invalid min_fit value: use perfect|good|marginal|too_tight"
|
||||
}
|
||||
```
|
||||
|
||||
Server errors return HTTP 500 with `{"error": "..."}`.
|
||||
|
||||
## Client integration recommendations
|
||||
|
||||
### 1) Polling pattern for schedulers
|
||||
For each node agent:
|
||||
1. Call `/health`.
|
||||
2. Call `/api/v1/system`.
|
||||
3. Call `/api/v1/models/top?limit=K&min_fit=good`.
|
||||
4. Attach node metadata and forward to your central scheduler.
|
||||
|
||||
### 2) Conservative placement defaults
|
||||
For production placement, prefer:
|
||||
|
||||
```text
|
||||
min_fit=good
|
||||
include_too_tight=false
|
||||
sort=score
|
||||
limit=5..20
|
||||
```
|
||||
|
||||
### 3) Per-workload targeting
|
||||
Examples:
|
||||
- Coding workloads: `use_case=coding`
|
||||
- Embedding workloads: `use_case=embedding`
|
||||
- Runtime constrained to llama.cpp fleet: `runtime=llamacpp`
|
||||
|
||||
### 4) Stable parsing
|
||||
Treat unknown fields as forward-compatible additions:
|
||||
- Parse required fields you depend on.
|
||||
- Ignore unknown fields.
|
||||
|
||||
## Curl examples
|
||||
|
||||
```sh
|
||||
curl http://127.0.0.1:8787/health
|
||||
curl http://127.0.0.1:8787/api/v1/system
|
||||
curl "http://127.0.0.1:8787/api/v1/models?limit=20&min_fit=marginal&sort=score"
|
||||
curl "http://127.0.0.1:8787/api/v1/models/top?limit=5&min_fit=good&use_case=coding"
|
||||
curl "http://127.0.0.1:8787/api/v1/models/Mistral?runtime=any"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## MCP Server Mode
|
||||
|
||||
llmfit can run as an MCP (Model Context Protocol) server over stdio, making it discoverable by AI agents (Claude, Cursor, etc.).
|
||||
|
||||
### Start the MCP server
|
||||
|
||||
```sh
|
||||
llmfit serve --mcp
|
||||
```
|
||||
|
||||
Global hardware overrides still apply:
|
||||
|
||||
```sh
|
||||
llmfit --memory 24G --ram 64G serve --mcp
|
||||
```
|
||||
|
||||
### MCP client configuration
|
||||
|
||||
Add to your MCP client config (e.g. `claude_desktop_config.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"llmfit": {
|
||||
"command": "llmfit",
|
||||
"args": ["serve", "--mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Available tools
|
||||
|
||||
| Tool | Description | Parameters |
|
||||
|------|-------------|------------|
|
||||
| `get_system_specs` | Node hardware info (RAM, GPU, CPU) | None |
|
||||
| `recommend_models` | Top models for this hardware | `limit?`, `use_case?`, `min_fit?`, `runtime?`, `license?`, `sort?` |
|
||||
| `search_models` | Free-text model search | `query`, `limit?` |
|
||||
| `plan_hardware` | Hardware requirements for a model | `model`, `context?`, `quant?`, `target_tps?` |
|
||||
| `get_runtimes` | Installed inference runtimes | None |
|
||||
| `get_installed_models` | Models in local runtimes | None |
|
||||
|
||||
---
|
||||
|
||||
## NATS Event Publishing
|
||||
|
||||
When built with the `nats` feature, llmfit can publish hardware and model events to NATS for integration with coordination systems (e.g. Sympozium membrane).
|
||||
|
||||
### Build with NATS support
|
||||
|
||||
```sh
|
||||
cargo build --features nats
|
||||
```
|
||||
|
||||
### Enable event publishing
|
||||
|
||||
```sh
|
||||
llmfit serve --send-events --nats-url nats://localhost:4222
|
||||
llmfit serve --mcp --send-events # also works with MCP mode
|
||||
```
|
||||
|
||||
The `NATS_URL` environment variable is also supported.
|
||||
|
||||
### Event subjects
|
||||
|
||||
Events are published to `llmfit.{event_type}.{hostname}`:
|
||||
|
||||
| Subject | Trigger | Payload |
|
||||
|---------|---------|---------|
|
||||
| `llmfit.system.{hostname}` | Startup + every 60s | System hardware specs |
|
||||
| `llmfit.fit.{hostname}` | After fit analysis | Model fit summary |
|
||||
| `llmfit.plan.{hostname}` | After plan estimate | Plan estimate |
|
||||
| `llmfit.runtimes.{hostname}` | Startup + on query | Runtime availability |
|
||||
| `llmfit.installed.{hostname}` | Startup + on query | Installed models |
|
||||
|
||||
### Event envelope
|
||||
|
||||
All events are wrapped in a common envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "1747058400",
|
||||
"hostname": "worker-1",
|
||||
"event_type": "system",
|
||||
"version": "1",
|
||||
"data": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
### Subscribe to events
|
||||
|
||||
```sh
|
||||
nats sub 'llmfit.>' # all events from all nodes
|
||||
nats sub 'llmfit.system.>' # system specs from all nodes
|
||||
nats sub 'llmfit.system.worker-1' # system specs from specific node
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Versioning notes
|
||||
|
||||
Current API prefix is `v1`.
|
||||
|
||||
If you build long-lived clients, pin to `/api/v1/...` and validate behavior with the local test script in `scripts/test_api.py`.
|
||||
@@ -0,0 +1,129 @@
|
||||
# Audio Model Support — Implementation Plan
|
||||
|
||||
This document describes the Rust changes needed to fully support
|
||||
`pipeline_tag: automatic-speech-recognition` models (Whisper variants).
|
||||
|
||||
The JSON data additions (`llmfit-core/data/hf_models.json`) in this branch are ready.
|
||||
The Rust integration changes below are the next step — open for discussion.
|
||||
|
||||
## Data changes (this branch)
|
||||
|
||||
`llmfit-core/data/hf_models.json` — 4 new entries with:
|
||||
- `pipeline_tag: "automatic-speech-recognition"`
|
||||
- `capabilities: ["audio"]`
|
||||
- New fields (custom, don't break existing Rust deserialization via `#[serde(default)]`):
|
||||
- `_audio_rtf_gpu: f64` — Real-Time Factor on GPU (0.007 = 7x realtime)
|
||||
- `_audio_rtf_cpu: f64` — RTF on CPU
|
||||
- `_audio_vram_gb: f64` — VRAM needed at F16
|
||||
- `_audio_backends: [str]` — supported servers
|
||||
|
||||
## Rust changes needed
|
||||
|
||||
### 1. `llmfit-core/src/models.rs`
|
||||
|
||||
Add `Capability::Audio` to the `Capability` enum:
|
||||
|
||||
```rust
|
||||
pub enum Capability {
|
||||
Vision,
|
||||
ToolUse,
|
||||
Reasoning,
|
||||
Embedding,
|
||||
Audio, // ← new
|
||||
}
|
||||
```
|
||||
|
||||
Extend `LlmModel` deserialization to accept the new `_audio_*` fields:
|
||||
|
||||
```rust
|
||||
// Inside LlmModel or a companion AudioMeta struct
|
||||
#[serde(default)]
|
||||
pub audio_rtf_gpu: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub audio_rtf_cpu: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub audio_vram_gb: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub audio_backends: Vec<String>,
|
||||
```
|
||||
|
||||
Add `UseCase::Audio` variant and detect it from `pipeline_tag`:
|
||||
|
||||
```rust
|
||||
pub enum UseCase {
|
||||
General, Coding, Reasoning, Chat, Multimodal, Embedding,
|
||||
Audio, // ← new
|
||||
}
|
||||
|
||||
impl UseCase {
|
||||
pub fn from_model(model: &LlmModel) -> Self {
|
||||
// existing checks …
|
||||
if model.pipeline_tag.as_deref() == Some("automatic-speech-recognition")
|
||||
|| model.capabilities.contains(&Capability::Audio)
|
||||
{
|
||||
UseCase::Audio
|
||||
} else { /* existing logic */ }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. `llmfit-core/src/fit.rs`
|
||||
|
||||
Audio models don't use tok/s — they use RTF (Real-Time Factor).
|
||||
Add an `AudioFit` struct separate from `ModelFit`:
|
||||
|
||||
```rust
|
||||
pub struct AudioFit {
|
||||
pub model: LlmModel,
|
||||
pub rtf_gpu: Option<f64>,
|
||||
pub rtf_cpu: f64,
|
||||
pub fits_vram: bool,
|
||||
pub fits_ram: bool,
|
||||
pub recommended_backend: String,
|
||||
}
|
||||
```
|
||||
|
||||
Scoring for audio: `score = accuracy_tier - latency_penalty - vram_penalty`.
|
||||
Lower RTF = faster = better score.
|
||||
|
||||
### 3. `llmfit-core/src/providers.rs`
|
||||
|
||||
Add Whisper server provider detection:
|
||||
|
||||
```rust
|
||||
/// mlx-openai-server Whisper endpoint (Apple Silicon path).
|
||||
pub struct MlxWhisperProvider;
|
||||
impl ModelProvider for MlxWhisperProvider {
|
||||
fn check_running(&self) -> Option<ProviderInfo> {
|
||||
probe_http("http://localhost:18000/v1/audio/transcriptions")
|
||||
.map(|_| ProviderInfo { name: "mlx-openai-server", port: 18000 })
|
||||
}
|
||||
}
|
||||
|
||||
/// faster-whisper-server (Docker, NVIDIA/CPU path).
|
||||
pub struct FasterWhisperProvider;
|
||||
impl ModelProvider for FasterWhisperProvider {
|
||||
fn check_running(&self) -> Option<ProviderInfo> {
|
||||
probe_http("http://localhost:8000/health")
|
||||
.map(|_| ProviderInfo { name: "faster-whisper-server", port: 8000 })
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. `llmfit-tui/src/main.rs` / CLI
|
||||
|
||||
Add `llmfit fit --kind audio` / `llmfit recommend --kind audio` to filter
|
||||
to ASR models only (useful for the TLDR smart installer use case).
|
||||
|
||||
```bash
|
||||
llmfit --json fit --kind audio -n 3
|
||||
```
|
||||
|
||||
## Why this matters
|
||||
|
||||
Projects like [TLDR](https://github.com/melnikaite/tldr-free) (Chrome extension
|
||||
that summarizes pages/videos) use an OpenAI-compatible Whisper backend for
|
||||
audio transcription. Choosing the right Whisper model for your hardware is
|
||||
just as confusing as choosing an LLM — RTF on a GTX 1660 Ti vs. Apple M3 Pro
|
||||
is wildly different. This brings llmfit's hardware-aware recommendations to
|
||||
the audio domain.
|
||||
+856
@@ -0,0 +1,856 @@
|
||||
# Changelog
|
||||
|
||||
## [1.1.2](https://github.com/AlexsJones/llmfit/compare/v1.1.1...v1.1.2) (2026-07-10)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **share:** ship the registered OAuth App client id — interactive login enabled ([a94ffcb](https://github.com/AlexsJones/llmfit/commit/a94ffcbe9412f0974224c87290ae01f89d01e009))
|
||||
* **share:** ship the registered OAuth App client id — interactive login enabled ([325f8a3](https://github.com/AlexsJones/llmfit/commit/325f8a3ff34adf0772990335e93f1efad212d735))
|
||||
|
||||
## [1.1.1](https://github.com/AlexsJones/llmfit/compare/v1.1.0...v1.1.1) (2026-07-10)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **bench:** ship merged community benchmarks to every user in the next release ([f2fd476](https://github.com/AlexsJones/llmfit/commit/f2fd47672ff1b81bacdba3169e163f235a32070a))
|
||||
* **bench:** ship merged community benchmarks to every user in the next release ([5c46f92](https://github.com/AlexsJones/llmfit/commit/5c46f92a2411edbfa7a829cc0e4c92f355189d76))
|
||||
|
||||
## [1.1.0](https://github.com/AlexsJones/llmfit/compare/v1.0.1...v1.1.0) (2026-07-10)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* bench --share — contribute benchmarks as PRs without the gh CLI (RFC [#710](https://github.com/AlexsJones/llmfit/issues/710)) ([5114694](https://github.com/AlexsJones/llmfit/commit/51146943d75d09faeb7cc7412f32984638679ee7))
|
||||
* **bench:** add --share to contribute benchmarks via PR without gh CLI ([721eb66](https://github.com/AlexsJones/llmfit/commit/721eb666ab4c0fc15e3ea75a510aacb127d04a5d))
|
||||
* **bench:** detect llama-server as a first-class provider ([6dc367b](https://github.com/AlexsJones/llmfit/commit/6dc367be97329be00301a8aeeb63fc753d0c9ac2))
|
||||
* **bench:** local benchmark store — save runs, share the backlog later ([0e22a75](https://github.com/AlexsJones/llmfit/commit/0e22a75abc2da4da0f8c51797f79f391fc74cd3c))
|
||||
* **bench:** local store follow-ups — share backlog, dedup PRs, local-measurement overrides + calibration ([390cbf1](https://github.com/AlexsJones/llmfit/commit/390cbf1c6726580c308cc9113df0eede7ae7dba1))
|
||||
* **fit:** local benchmark runs override estimated tok/s ([274c048](https://github.com/AlexsJones/llmfit/commit/274c048bc3366bc74851128062cce27e81287d88))
|
||||
* **fit:** purge test-stub catalog entries; calibrate estimates from local benchmarks ([fb049c2](https://github.com/AlexsJones/llmfit/commit/fb049c210c13f65c0d77ac8da0ce50aec7c0b332))
|
||||
* **share:** reuse open benchmark PR and make submissions idempotent ([52f3afb](https://github.com/AlexsJones/llmfit/commit/52f3afb10b6fb521d880bae896cae15b117d9c69))
|
||||
* **tui:** expose installed/GGUF availability filter in the Filter modal ([dd940d5](https://github.com/AlexsJones/llmfit/commit/dd940d5288980c8ee9cac5724d4a75318d4be9d3))
|
||||
* **tui:** filter by installed models in the Filter modal ([75f4f1e](https://github.com/AlexsJones/llmfit/commit/75f4f1ec773435288b59ff83e699091bef253820))
|
||||
* **tui:** offer to benchmark the selected model on entering leaderboard ([61f6a47](https://github.com/AlexsJones/llmfit/commit/61f6a4705c5b72e0838aa957ef4507efde86bfaa))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **data:** verify GGUF source provenance; unlink wrong-model quants ([63b2d3a](https://github.com/AlexsJones/llmfit/commit/63b2d3aada13d87b276666497074c8c816d8805b))
|
||||
* **providers:** stop marking whole model families as installed for llama.cpp ([f839633](https://github.com/AlexsJones/llmfit/commit/f8396335b0fe748555e0764f0386ab7e17fac73e))
|
||||
* **tests:** pin GitHub-credential state in bench-offer toggle test ([618ba16](https://github.com/AlexsJones/llmfit/commit/618ba162a146b623711202264a3405695be1a037))
|
||||
* **tests:** pin GitHub-credential state in bench-offer toggle test ([fb5caff](https://github.com/AlexsJones/llmfit/commit/fb5caffc371907d53578c1919bf6ffa5a1616b11))
|
||||
|
||||
## [1.0.1](https://github.com/AlexsJones/llmfit/compare/v1.0.0...v1.0.1) (2026-07-09)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* bump internal llmfit-core version requirement to 1.0.0 ([7274a45](https://github.com/AlexsJones/llmfit/commit/7274a450a04107b3272cf958a5f20e18a516a406))
|
||||
|
||||
## [1.0.0](https://github.com/AlexsJones/llmfit/compare/v0.9.38...v1.0.0) (2026-07-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add global --tui flag to force interactive TUI ([749b22f](https://github.com/AlexsJones/llmfit/commit/749b22fcd30d72fb2c57a1b1c91af3d834c68c3f))
|
||||
* add global --tui flag to force interactive TUI ([#547](https://github.com/AlexsJones/llmfit/issues/547)) ([f0146b1](https://github.com/AlexsJones/llmfit/commit/f0146b1b4e3d1233fe9a3dc4dcbcc84f5aa6ca83))
|
||||
* add model language metadata ([784af4d](https://github.com/AlexsJones/llmfit/commit/784af4d8c17c6c16762c0ca9fb1f0a19072916a3))
|
||||
* add TTS model discovery ([40a69f6](https://github.com/AlexsJones/llmfit/commit/40a69f62697a5d92af38f8f24d337b3b2eac648f))
|
||||
* auto-detect system RAM bandwidth for MoE-offload estimates ([c10f567](https://github.com/AlexsJones/llmfit/commit/c10f567a7c355e5d52586a44d778b7553fed2bfd))
|
||||
* auto-detect system RAM bandwidth for MoE-offload estimates ([f469543](https://github.com/AlexsJones/llmfit/commit/f469543c9c13b762748af20c60ec9fb9ea971c3d))
|
||||
* discover TTS language metadata in scraper ([74cd130](https://github.com/AlexsJones/llmfit/commit/74cd1306197000a8573cb102a51e4ca47262d923))
|
||||
* estimate basis + measured tok/s provenance ([#292](https://github.com/AlexsJones/llmfit/issues/292)) ([93f6806](https://github.com/AlexsJones/llmfit/commit/93f6806fb0cd50fec555ab0166acc71f1d483231))
|
||||
* expose estimate basis and local verification commands ([#292](https://github.com/AlexsJones/llmfit/issues/292)) ([af54caf](https://github.com/AlexsJones/llmfit/commit/af54caf3a35c85f394df2c22ba26c9c6b6b5a98c))
|
||||
* pin Huihui-Qwen3.6-35B-A3B-abliterated and add GGUF source ([d048900](https://github.com/AlexsJones/llmfit/commit/d048900953dc4196a5bf59a4af692558cb5cc282))
|
||||
* pin Huihui-Qwen3.6-35B-A3B-abliterated and add GGUF source ([#656](https://github.com/AlexsJones/llmfit/issues/656)) ([e0a5654](https://github.com/AlexsJones/llmfit/commit/e0a56545387b05f4f966a0e8e5d07c0c66a14276))
|
||||
* surface community-measured tok/s when hardware matches benchmark data ([34eca7e](https://github.com/AlexsJones/llmfit/commit/34eca7e8432c42ccf0fb20a36b1e839f827dfb80))
|
||||
* surface community-measured tok/s when hardware matches benchmark data ([730cc5d](https://github.com/AlexsJones/llmfit/commit/730cc5d1e404a672671c9d2633866d1557c38002))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* harden AMD sysfs GPU detection against silent card drops ([#303](https://github.com/AlexsJones/llmfit/issues/303), [#638](https://github.com/AlexsJones/llmfit/issues/638)) ([84a6cfa](https://github.com/AlexsJones/llmfit/commit/84a6cfa7c430906d77b8803fec28e5c362461a3c))
|
||||
* harden AMD sysfs GPU detection against silent card drops ([#303](https://github.com/AlexsJones/llmfit/issues/303), [#638](https://github.com/AlexsJones/llmfit/issues/638)) ([4dc94ca](https://github.com/AlexsJones/llmfit/commit/4dc94ca3fca3601dcbf040128ae4ed9b699875e3))
|
||||
* keep text generation discovery budget ([5b55e2c](https://github.com/AlexsJones/llmfit/commit/5b55e2c5710f28b2ff79e9cde4ba4f94df6f9fd2))
|
||||
* mark TTS models as unsupported runtime ([5ca524e](https://github.com/AlexsJones/llmfit/commit/5ca524e6ea8a23d93c4a130971f99f837c82226d))
|
||||
* **ollama:** don't mark cloud models as locally installed (closes [#619](https://github.com/AlexsJones/llmfit/issues/619)) ([580afee](https://github.com/AlexsJones/llmfit/commit/580afee47da41837d27f51cadfba99e1cbed4022))
|
||||
* preserve popularity metadata through scraper merge and discovery ([b2b35d6](https://github.com/AlexsJones/llmfit/commit/b2b35d60d15e93953e73061a336ad0368c04572c))
|
||||
* resolve [#171](https://github.com/AlexsJones/llmfit/issues/171) — Include ONNX models. ([9939e67](https://github.com/AlexsJones/llmfit/commit/9939e67b309b24252e1838bdc231d3a1d0885704))
|
||||
* support TTS catalog metadata ([c98b3d5](https://github.com/AlexsJones/llmfit/commit/c98b3d5fa72418b65a1e84c5cb5d1eb8a310b857))
|
||||
* use cross-platform which crate for binary detection in TUI ([55d77ec](https://github.com/AlexsJones/llmfit/commit/55d77ecd0f66f670641ead6da19ffd8fec2bcf3c))
|
||||
* use cross-platform which crate for binary detection in TUI ([d450daa](https://github.com/AlexsJones/llmfit/commit/d450daac08313145687d366d1cc6dc4db57046fe))
|
||||
* wire onnx catalog into model database ([f2e7dba](https://github.com/AlexsJones/llmfit/commit/f2e7dbafce1aba08e76d18494c1e556141ac2a13))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* cut 1.0.0 ([21ebc84](https://github.com/AlexsJones/llmfit/commit/21ebc847dede0eb05b214296c075d8d2f21c51c5))
|
||||
|
||||
## [0.9.38](https://github.com/AlexsJones/llmfit/compare/v0.9.37...v0.9.38) (2026-07-04)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **docker:** drop COPY of removed root data/ directory ([a4445c7](https://github.com/AlexsJones/llmfit/commit/a4445c70885ee399a5b8e39b4ded938a09230064))
|
||||
* **docker:** drop COPY of removed root data/ directory ([5bb514d](https://github.com/AlexsJones/llmfit/commit/5bb514d130964311617269b0cffd4663a31522e3))
|
||||
|
||||
## [0.9.37](https://github.com/AlexsJones/llmfit/compare/v0.9.36...v0.9.37) (2026-07-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **calibration:** weekly benchmark refresh + estimate-accuracy audit ([#112](https://github.com/AlexsJones/llmfit/issues/112)/[#119](https://github.com/AlexsJones/llmfit/issues/119)) ([5548723](https://github.com/AlexsJones/llmfit/commit/55487236685fde60a2be09580fe892daeb6a9b8b))
|
||||
* **calibration:** weekly benchmark refresh + estimate-accuracy audit against measured data ([6c40dcf](https://github.com/AlexsJones/llmfit/commit/6c40dcf3127ada85a47cb4023c41fee28d33ec3e))
|
||||
* **claim:** JSON output for programmatic consumers (llmfit-dra M0) ([f469c22](https://github.com/AlexsJones/llmfit/commit/f469c22cb48cc8e15b33119cd7cf166474ab50a7))
|
||||
* **claim:** machine-readable JSON output via the global --json flag ([1636e51](https://github.com/AlexsJones/llmfit/commit/1636e51bb8b452b6c9ab8e47f38e0ac764f25f2f))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **calibration:** exclude speculative-decoding/MTP runs from the accuracy audit ([1c058aa](https://github.com/AlexsJones/llmfit/commit/1c058aa22bd57456487a15aa618155fd1340ef26))
|
||||
|
||||
## [0.9.36](https://github.com/AlexsJones/llmfit/compare/v0.9.35...v0.9.36) (2026-07-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add 'llmfit doctor' hardware diagnostic report for bug reports ([53663bc](https://github.com/AlexsJones/llmfit/commit/53663bcb9935d79c6f959b357d99d622d6cbcef1))
|
||||
* add `llmfit doctor` hardware diagnostic report ([bcd7dc0](https://github.com/AlexsJones/llmfit/commit/bcd7dc0c89b48909cfdd0ce20813bd58a2e84aeb))
|
||||
* add issue templates for bug reports ([bab1fe9](https://github.com/AlexsJones/llmfit/commit/bab1fe9166a037aac1e3d4da10484e0f91812bc1))
|
||||
* request 'llmfit doctor' output in the bug report template ([b4069d9](https://github.com/AlexsJones/llmfit/commit/b4069d927b9f7d271664d9e24324b3fe5716fa29))
|
||||
* **scoring:** benchmark-informed task alignment per use case ([ce6d37e](https://github.com/AlexsJones/llmfit/commit/ce6d37e034b3978e687a00589ac7ce10e81ae7e9))
|
||||
* **scoring:** benchmark-informed task alignment per use case (issue [#150](https://github.com/AlexsJones/llmfit/issues/150)) ([c994615](https://github.com/AlexsJones/llmfit/commit/c994615357f1b2577ba5096d7d176cf928f27a56))
|
||||
* serve over a Unix domain socket (--unix-socket) ([de62c2b](https://github.com/AlexsJones/llmfit/commit/de62c2bc7e2af432771e0036b35e5fb76a50ed7e))
|
||||
* serve over a Unix domain socket (--unix-socket) ([03d6ca5](https://github.com/AlexsJones/llmfit/commit/03d6ca5170df4f924b3bf24e94f27035fa26bb6a))
|
||||
* show usable context per model (native→achievable) in TUI, CLI, and JSON ([e33a3a3](https://github.com/AlexsJones/llmfit/commit/e33a3a31be5a7a7b3953c4f33a02732c1790353c)), closes [#621](https://github.com/AlexsJones/llmfit/issues/621)
|
||||
* usable-context display — native→achievable (closes [#621](https://github.com/AlexsJones/llmfit/issues/621)) ([1706929](https://github.com/AlexsJones/llmfit/commit/1706929ea367fd137117173f56827640bc04404c))
|
||||
* user-local custom model overlay (closes [#451](https://github.com/AlexsJones/llmfit/issues/451)) ([decef82](https://github.com/AlexsJones/llmfit/commit/decef827b3785df9a5c785b49ed128f98d5c3aaa))
|
||||
* user-local custom model overlay (custom_models.json) ([e55c8de](https://github.com/AlexsJones/llmfit/commit/e55c8dec388a1788b6093e0ef4c1091149cfe3d9)), closes [#451](https://github.com/AlexsJones/llmfit/issues/451)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* guard optional attribute lookups in generated claim CEL ([c6f481f](https://github.com/AlexsJones/llmfit/commit/c6f481f27557d3e0fd6fc584fd1b1fa008ff95a4))
|
||||
* guard optional attribute lookups in generated claim CEL ([a6f3c6a](https://github.com/AlexsJones/llmfit/commit/a6f3c6a6bc11e3a4c141dc7e0c3891263958a712))
|
||||
* **hardware:** correct Intel GPU detection — iGPUs get the unified RAM pool, dGPUs get Vulkan VRAM ([4834767](https://github.com/AlexsJones/llmfit/commit/4834767462a7ac939ddb0306796e9661913f8310))
|
||||
* **hardware:** detect generic-named MI50-class accelerators (issue [#638](https://github.com/AlexsJones/llmfit/issues/638)) ([180f484](https://github.com/AlexsJones/llmfit/commit/180f4844630e4ca08e8971482a615564d5be64f8))
|
||||
* **hardware:** don't drop large-VRAM AMD accelerators with generic ROCm names ([bf16eb8](https://github.com/AlexsJones/llmfit/commit/bf16eb8e87b31e1c0f5c6154e82f7b8967d54085))
|
||||
* **hardware:** Intel iGPU unified-memory pool + discrete Arc VRAM via Vulkan (issue [#609](https://github.com/AlexsJones/llmfit/issues/609)) ([a120aa3](https://github.com/AlexsJones/llmfit/commit/a120aa38d9410b2d874bbe7c70653b4f2c1920b7))
|
||||
* **schema:** add 'audio' to capability enum ([a812232](https://github.com/AlexsJones/llmfit/commit/a812232f0eb02dc66544e508fe6e5e587acbd1f0))
|
||||
|
||||
## [0.9.35](https://github.com/AlexsJones/llmfit/compare/v0.9.34...v0.9.35) (2026-07-02)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add claim subcommand generating Kubernetes DRA ResourceClaims ([08547ef](https://github.com/AlexsJones/llmfit/commit/08547efa277dd3f733ecdf72312a3376fbc2a01b))
|
||||
* add claim subcommand generating Kubernetes DRA ResourceClaims ([3369a49](https://github.com/AlexsJones/llmfit/commit/3369a492630f357c65e2344de18b5a2e35ed4950))
|
||||
* expose GPU memory bandwidth in system JSON and add Strix Halo entries ([3bfd334](https://github.com/AlexsJones/llmfit/commit/3bfd33406b071f48fddc7fb6e99fe62e55096511))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* reset list cursor to top when the search query changes ([5e4dc7b](https://github.com/AlexsJones/llmfit/commit/5e4dc7b1e578c5a2366a8f7b94220d68178e2fae))
|
||||
* reset list cursor to top when the search query changes ([43884f5](https://github.com/AlexsJones/llmfit/commit/43884f500a9b392b03bc2d2bc539ba6430d954fd)), closes [#657](https://github.com/AlexsJones/llmfit/issues/657)
|
||||
|
||||
## [0.9.34](https://github.com/AlexsJones/llmfit/compare/v0.9.33...v0.9.34) (2026-06-28)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* pin Docker builder to bookworm to match runtime glibc ([f13a633](https://github.com/AlexsJones/llmfit/commit/f13a6332e62713b22bf96b7a26b551f43caed831))
|
||||
* pin Docker builder to bookworm to match runtime glibc ([48caf17](https://github.com/AlexsJones/llmfit/commit/48caf178baaf7e7c0f81bfd30f4972b404528cf7)), closes [#655](https://github.com/AlexsJones/llmfit/issues/655)
|
||||
|
||||
## [0.9.33](https://github.com/AlexsJones/llmfit/compare/v0.9.32...v0.9.33) (2026-06-23)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **docker:** bump builder image to rust 1.95 (sysinfo MSRV) ([b63a142](https://github.com/AlexsJones/llmfit/commit/b63a14244d93fb1fdd50b2f49c96abaac872b977))
|
||||
* **docker:** bump builder image to rust 1.95 (sysinfo MSRV) ([b943f3a](https://github.com/AlexsJones/llmfit/commit/b943f3a84a63e77a740acacd38a09c69774dda8c))
|
||||
|
||||
## [0.9.32](https://github.com/AlexsJones/llmfit/compare/v0.9.31...v0.9.32) (2026-06-23)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add NCAI VAETKI models to curated model database ([b3c04a8](https://github.com/AlexsJones/llmfit/commit/b3c04a82055875b27a8808131df8f41e544ca759))
|
||||
* **analysis:** add InstalledIndex and build_model_fits to consolidate provider detection ([33dce8b](https://github.com/AlexsJones/llmfit/commit/33dce8bbb9f8a527c48fec07152fa2c3a0983784))
|
||||
* **data:** add Whisper/ASR model entries + audio support roadmap ([27bae48](https://github.com/AlexsJones/llmfit/commit/27bae4878f798c7069f936de6fa9ed5b1dcc3312))
|
||||
* **python:** add riscv64 wheel build ([e12bb85](https://github.com/AlexsJones/llmfit/commit/e12bb85abdd752e632930c3086a4afc935e9a4a6))
|
||||
* **python:** add riscv64 wheel build ([cb377f4](https://github.com/AlexsJones/llmfit/commit/cb377f45595b745d7bf0963ab1c86b807c210b01))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* bundle WebView2 bootstrapper for Windows desktop app ([c72103b](https://github.com/AlexsJones/llmfit/commit/c72103b784134b8cc66725d4e8e9c36b0d01e64d))
|
||||
* **fit:** CPU-only models that fit with headroom reach Good, not Marginal ([aad4c37](https://github.com/AlexsJones/llmfit/commit/aad4c373285e328bcbab72577ee49d62cca1515d))
|
||||
* **fit:** CPU-only models that fit with headroom reach Good, not Marginal ([0ced1b7](https://github.com/AlexsJones/llmfit/commit/0ced1b775cb39f783cabf4411d25e1e1aa188015))
|
||||
* **hardware:** detect all GPUs across sysfs, ROCm tabular, and fit pool ([0989ca2](https://github.com/AlexsJones/llmfit/commit/0989ca2dc443f782cf34cec709e989c23f234178))
|
||||
* **hardware:** ROCm tabular multi-GPU parsing + summed fit VRAM pool ([4e27617](https://github.com/AlexsJones/llmfit/commit/4e27617baaeaf1902131f59c71763de360ef6837))
|
||||
* **scoring:** MoE active-param quality, recency bonus, smooth fit curve ([c499c00](https://github.com/AlexsJones/llmfit/commit/c499c000948cb620564d9c8433df5b8b5d36880d))
|
||||
* **scoring:** MoE active-param quality, recency bonus, smooth fit curve ([2db3e55](https://github.com/AlexsJones/llmfit/commit/2db3e552ad637437589e692bf91bc9ca69350925))
|
||||
* **scoring:** treat headroom as a perfect fit (one-sided fit curve) ([235be33](https://github.com/AlexsJones/llmfit/commit/235be33b1d8ada30693f72b48b48d50bd48df5a9))
|
||||
* **tui:** make provider filter input cursor-aware and ASCII-only ([933644e](https://github.com/AlexsJones/llmfit/commit/933644e4d1a80969cb7bfc1c5386423291a50132))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* **analysis:** parallelize provider detection in detect_all with thread::scope ([d0660bc](https://github.com/AlexsJones/llmfit/commit/d0660bc72e601a7707caabb2f4e042a0701c491e))
|
||||
|
||||
## [0.9.31](https://github.com/AlexsJones/llmfit/compare/v0.9.30...v0.9.31) (2026-06-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **lmstudio:** Add LMSTUDIO_API_KEY support for authenticated instances ([0dc2712](https://github.com/AlexsJones/llmfit/commit/0dc2712df4b4d947e0db371c98bf376d4da16c85))
|
||||
* **lmstudio:** Add LMSTUDIO_API_KEY support for authenticated instances ([302cb53](https://github.com/AlexsJones/llmfit/commit/302cb535830e7e001c0735596d0c06da96fdc467))
|
||||
* speed up provider navigation when holding shift (gh-571) ([f0d4783](https://github.com/AlexsJones/llmfit/commit/f0d4783dd99f4c12a0fe15d564d8bc787ef7c755))
|
||||
* speed up provider navigation when holding shift (gh-571) ([7d74d7d](https://github.com/AlexsJones/llmfit/commit/7d74d7d4a93cb8c229868b00d8bc3385005e3551))
|
||||
* upgrade MiniMax default model to M3 ([d45a546](https://github.com/AlexsJones/llmfit/commit/d45a546ebd9eca78cb5e09f191321e67807112d2))
|
||||
* upgrade MiniMax default model to M3 ([7407d10](https://github.com/AlexsJones/llmfit/commit/7407d1035ef7dd7b14ed50a1dc1f3f6e21a08a44))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add Gemma 4 QAT models and fallback HF search ([e40d3b8](https://github.com/AlexsJones/llmfit/commit/e40d3b8e00b1db1fae9234e34fadde8a8a3945a2))
|
||||
* add Gemma 4 QAT models and fallback HF search ([9b4582f](https://github.com/AlexsJones/llmfit/commit/9b4582f0e45c02ffb09abaef649cc45e01344f63)), closes [#613](https://github.com/AlexsJones/llmfit/issues/613)
|
||||
* **lmstudio:** empty string guard, README docs, and unit test for LMSTUDIO_API_KEY ([46f70b7](https://github.com/AlexsJones/llmfit/commit/46f70b790e3bd177e001469d04456ce1e20193c5))
|
||||
* replace unsafe env mutation in tests with pure logic tests ([33047d8](https://github.com/AlexsJones/llmfit/commit/33047d85fab2913d09905922df9a3daef6b5a314))
|
||||
* **tui:** handle search input cursor and overflow ([66f4c88](https://github.com/AlexsJones/llmfit/commit/66f4c887b37c617944a8bc14a1d0a20aedac9767))
|
||||
* **tui:** handle search input cursor and overflow ([8596005](https://github.com/AlexsJones/llmfit/commit/85960054bbb88c961ffa4797006dd3475a8b4699))
|
||||
* **tui:** handle unicode download directory input ([b34d164](https://github.com/AlexsJones/llmfit/commit/b34d164cb9f5baf60f3b14058599d0d13755e212))
|
||||
* **tui:** handle unicode download directory input ([da120f8](https://github.com/AlexsJones/llmfit/commit/da120f8d6993dd2e6cfed899478b6796b1992221))
|
||||
* **tui:** ignore modified key input in search ([6c0b697](https://github.com/AlexsJones/llmfit/commit/6c0b69701e8ec9be28a7a98bd0e94812f64a037c))
|
||||
* **tui:** ignore modified key input in search ([8de0a1b](https://github.com/AlexsJones/llmfit/commit/8de0a1b5a0627327f82360f9c9ced2ca71cfc4a7))
|
||||
* use GGUF metadata for param counts in GGUF-only repos ([3f9b22b](https://github.com/AlexsJones/llmfit/commit/3f9b22bb032a3b4444750ddfca12acfdf96e27df))
|
||||
* use GGUF metadata for parameter counts in GGUF-only repos ([ff28cdd](https://github.com/AlexsJones/llmfit/commit/ff28cdd002f4777ab1857b0c6e669c3c7789a1cd)), closes [#622](https://github.com/AlexsJones/llmfit/issues/622)
|
||||
* wrap env::set_var/remove_var in unsafe blocks for Rust 2024 edition ([55b7979](https://github.com/AlexsJones/llmfit/commit/55b79793d1aa8bd3ca6e6ab7d4f11c5f20f897f6))
|
||||
|
||||
## [0.9.30](https://github.com/AlexsJones/llmfit/compare/v0.9.29...v0.9.30) (2026-06-01)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **tui:** fuzzy search in provider filter popup ([#600](https://github.com/AlexsJones/llmfit/issues/600)) ([b055a04](https://github.com/AlexsJones/llmfit/commit/b055a04beb50008c3d868aa7f43598ad6b614d33))
|
||||
|
||||
## [0.9.29](https://github.com/AlexsJones/llmfit/compare/v0.9.28...v0.9.29) (2026-05-27)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* open TUI with best-fit model preselected ([#582](https://github.com/AlexsJones/llmfit/issues/582)) ([834ee20](https://github.com/AlexsJones/llmfit/commit/834ee20ca2d3cb18fa10292afce6f9ba75acb9e5))
|
||||
* rescrape model catalog and expand Ollama mappings ([#573](https://github.com/AlexsJones/llmfit/issues/573)) ([c3e3f0d](https://github.com/AlexsJones/llmfit/commit/c3e3f0d03fb9a83ced7a4c4ff0ba2ad6926db153))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* align SortColumn::next with display order of columns in TUI ([#576](https://github.com/AlexsJones/llmfit/issues/576)) ([9e3477e](https://github.com/AlexsJones/llmfit/commit/9e3477e2165f0b379a003c64a9651fcfd54de6db))
|
||||
* **core:** scan XDG cache path for HF models on macOS ([#568](https://github.com/AlexsJones/llmfit/issues/568)) ([#575](https://github.com/AlexsJones/llmfit/issues/575)) ([9ec7dd3](https://github.com/AlexsJones/llmfit/commit/9ec7dd35f7ab603e16caee163ed396f02a30a3ee))
|
||||
* **release:** unblock downstream publish jobs when sign-windows fails ([18b581f](https://github.com/AlexsJones/llmfit/commit/18b581fcf2fd26b50bd0b559a71c2adc86f953f3)), closes [#581](https://github.com/AlexsJones/llmfit/issues/581)
|
||||
* **release:** use always() to actually override transitive skip ([0d6aeb3](https://github.com/AlexsJones/llmfit/commit/0d6aeb3e245ec0574929717c8954f74a26120996))
|
||||
|
||||
## [0.9.28](https://github.com/AlexsJones/llmfit/compare/v0.9.27...v0.9.28) (2026-05-22)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **tui:** add ollama_name field to JSON output ([#566](https://github.com/AlexsJones/llmfit/issues/566)) ([36b3181](https://github.com/AlexsJones/llmfit/commit/36b3181ccd8d6343d06a0de05f3a3838cc055e50))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **core:** detect GGUF models in HuggingFace cache for llama.cpp ([#568](https://github.com/AlexsJones/llmfit/issues/568)) ([6093bcc](https://github.com/AlexsJones/llmfit/commit/6093bcc952eee0b5a64141809665c8bc823e21d8))
|
||||
* **tui:** include GGUF source providers in search and provider filter ([#569](https://github.com/AlexsJones/llmfit/issues/569)) ([8698b99](https://github.com/AlexsJones/llmfit/commit/8698b990aa73622e124c87ad2257fa32cae87e34))
|
||||
|
||||
## [0.9.27](https://github.com/AlexsJones/llmfit/compare/v0.9.26...v0.9.27) (2026-05-20)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **tui:** correct live-bench keybinding label from B to I ([7aae532](https://github.com/AlexsJones/llmfit/commit/7aae5326ef15018fccb9d1d9605252f75739a749))
|
||||
|
||||
## [0.9.26](https://github.com/AlexsJones/llmfit/compare/v0.9.25...v0.9.26) (2026-05-19)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **hardware:** detect distinct AMD GPU models in multi-GPU ROCm systems ([#559](https://github.com/AlexsJones/llmfit/issues/559)) ([a100552](https://github.com/AlexsJones/llmfit/commit/a100552a60876ad84ee3b092704a1f4b4dadc143)), closes [#550](https://github.com/AlexsJones/llmfit/issues/550)
|
||||
|
||||
## [0.9.25](https://github.com/AlexsJones/llmfit/compare/v0.9.24...v0.9.25) (2026-05-17)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **scoring:** add generation-aware quality scoring ([#553](https://github.com/AlexsJones/llmfit/issues/553)) ([a3e0eb2](https://github.com/AlexsJones/llmfit/commit/a3e0eb27c57fe05298292f4c61699b9aed2824ea))
|
||||
* **tui:** move provider detection to background threads for faster startup ([#554](https://github.com/AlexsJones/llmfit/issues/554)) ([9e8308d](https://github.com/AlexsJones/llmfit/commit/9e8308d3e6dccf83da35694233a53261902ebbf2))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **docker:** skip HTTP probe on Linux when Docker Desktop is not running ([#548](https://github.com/AlexsJones/llmfit/issues/548)) ([ce735c9](https://github.com/AlexsJones/llmfit/commit/ce735c95c39ab10bd936238b7ab611d336a13c12))
|
||||
|
||||
## [0.9.24](https://github.com/AlexsJones/llmfit/compare/v0.9.23...v0.9.24) (2026-05-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add MCP server mode and NATS event publishing to llmfit serve ([#545](https://github.com/AlexsJones/llmfit/issues/545)) ([c4958f7](https://github.com/AlexsJones/llmfit/commit/c4958f7edb59225ffd569779fd8442915720bf03))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **hardware:** fix iGPU filtering on Ryzen 9 9950X where 2 GB iGPU passes the >= 2 GB discrete GPU threshold ([#543](https://github.com/AlexsJones/llmfit/issues/543)) ([c655bc1](https://github.com/AlexsJones/llmfit/commit/c655bc1b2e10b5b0a13a123344755e450832a06d))
|
||||
* MoE offload speed overestimate for Qwen3-Coder-Next ([#544](https://github.com/AlexsJones/llmfit/issues/544)) ([5bdc9df](https://github.com/AlexsJones/llmfit/commit/5bdc9df9d8e28168fdc4650741be71b715a11fc6))
|
||||
|
||||
## [0.9.23](https://github.com/AlexsJones/llmfit/compare/v0.9.22...v0.9.23) (2026-05-10)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* populate architecture metadata for precise KV cache and MoE speed estimation ([#539](https://github.com/AlexsJones/llmfit/issues/539)) ([bab8c01](https://github.com/AlexsJones/llmfit/commit/bab8c012771acec3bbd4a051fd3720379bb01e5d))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **bench:** implement 5 follow-up improvements to PR [#278](https://github.com/AlexsJones/llmfit/issues/278) ([#523](https://github.com/AlexsJones/llmfit/issues/523)) ([91fee77](https://github.com/AlexsJones/llmfit/commit/91fee7702a5728fd1b0b64459c8be024fc82051a))
|
||||
|
||||
## [0.9.22](https://github.com/AlexsJones/llmfit/compare/v0.9.21...v0.9.22) (2026-05-06)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* overhaul model discovery with cursor pagination, trending, and additive DB ([#537](https://github.com/AlexsJones/llmfit/issues/537)) ([93e9d67](https://github.com/AlexsJones/llmfit/commit/93e9d673cf54bde62dc7b24a1400c1cd89f70c1c))
|
||||
|
||||
## [0.9.21](https://github.com/AlexsJones/llmfit/compare/v0.9.20...v0.9.21) (2026-05-05)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add AutoRound format detection to prevent misclassification as GGUF ([#532](https://github.com/AlexsJones/llmfit/issues/532)) ([8214b1f](https://github.com/AlexsJones/llmfit/commit/8214b1f19bad2ed8e5b9e9ae98d203447c09fd36))
|
||||
|
||||
## [0.9.20](https://github.com/AlexsJones/llmfit/compare/v0.9.19...v0.9.20) (2026-05-04)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **lmstudio:** resolve HF repo IDs to direct GGUF file URLs for download ([#530](https://github.com/AlexsJones/llmfit/issues/530)) ([115a01d](https://github.com/AlexsJones/llmfit/commit/115a01d66e1a1689715dc1266d98a5887cc3d687))
|
||||
|
||||
## [0.9.19](https://github.com/AlexsJones/llmfit/compare/v0.9.18...v0.9.19) (2026-05-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add llmfit bench for live inference benchmarking with quality metrics ([#278](https://github.com/AlexsJones/llmfit/issues/278)) ([8896fdb](https://github.com/AlexsJones/llmfit/commit/8896fdb861fa6950081c144ddf23d9eb24aef7ed))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **lmstudio:** wrap HuggingFace repo IDs in canonical URL for download ([#520](https://github.com/AlexsJones/llmfit/issues/520)) ([3399c70](https://github.com/AlexsJones/llmfit/commit/3399c70b6b2237c2285d92efe3ac1ebb2178b80e)), closes [#519](https://github.com/AlexsJones/llmfit/issues/519)
|
||||
* **ollama:** treat OLLAMA_HOST=0.0.0.0 as a bind address and fall back to localhost ([#521](https://github.com/AlexsJones/llmfit/issues/521)) ([3a7bcf5](https://github.com/AlexsJones/llmfit/commit/3a7bcf52628161f142c9eed6151c9408f5af8ef4)), closes [#516](https://github.com/AlexsJones/llmfit/issues/516)
|
||||
|
||||
## [0.9.18](https://github.com/AlexsJones/llmfit/compare/v0.9.17...v0.9.18) (2026-04-30)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **recommend:** add --output-llamacpp flag ([#508](https://github.com/AlexsJones/llmfit/issues/508)) ([7c5925c](https://github.com/AlexsJones/llmfit/commit/7c5925c949878fe1015d815a1245734c75db1aca))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* filter CPU Vulkan devices and suppress ROCm+Vulkan duplicates (fixes [#271](https://github.com/AlexsJones/llmfit/issues/271)) ([#328](https://github.com/AlexsJones/llmfit/issues/328)) ([fe8d3f0](https://github.com/AlexsJones/llmfit/commit/fe8d3f050a69b04589bf872e41001187fdd469ca))
|
||||
|
||||
## [0.9.17](https://github.com/AlexsJones/llmfit/compare/v0.9.16...v0.9.17) (2026-04-29)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add benchmark cache with offline fallback ([928390b](https://github.com/AlexsJones/llmfit/commit/928390b467a78e9bf7b1f73f819acdea8044184b))
|
||||
* add RWKV v7 G1f models (1.5B, 2.9B, 7.2B, 13.3B) ([#517](https://github.com/AlexsJones/llmfit/issues/517)) ([fab2bec](https://github.com/AlexsJones/llmfit/commit/fab2bec907db77ca63866a6c980bf8418d804b88)), closes [#515](https://github.com/AlexsJones/llmfit/issues/515)
|
||||
|
||||
## [0.9.16](https://github.com/AlexsJones/llmfit/compare/v0.9.15...v0.9.16) (2026-04-27)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add community benchmarks view from localmaxxing.com ([4d0ffa7](https://github.com/AlexsJones/llmfit/commit/4d0ffa79d8f332b51d5fbc990815612bf03b5069))
|
||||
* add hardware picker to benchmarks view (H key) ([21661b2](https://github.com/AlexsJones/llmfit/commit/21661b21fb28261502f915b62eb9691a5f837dcf))
|
||||
* add vLLM inference provider ([bfb0bb2](https://github.com/AlexsJones/llmfit/commit/bfb0bb202b97bc0fc97973b3688e54e83397a195)), closes [#501](https://github.com/AlexsJones/llmfit/issues/501)
|
||||
* **download:** add --output-dir flag to override cache path ([#499](https://github.com/AlexsJones/llmfit/issues/499)) ([f70409f](https://github.com/AlexsJones/llmfit/commit/f70409fc1c904100bc941c89aeea62b4fa8594b2))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add debug warning when util > 1.0 in GPU-mode VRAM pressure calculation ([#513](https://github.com/AlexsJones/llmfit/issues/513)) ([92d0f63](https://github.com/AlexsJones/llmfit/commit/92d0f63c54669f4381be32c75c2b86532b1b0618)), closes [#496](https://github.com/AlexsJones/llmfit/issues/496)
|
||||
* deduplicate AMD GPUs detected by both ROCm and Vulkan (fixes [#139](https://github.com/AlexsJones/llmfit/issues/139)) ([#314](https://github.com/AlexsJones/llmfit/issues/314)) ([43dd9fc](https://github.com/AlexsJones/llmfit/commit/43dd9fc075ec20fca1fcd1598c9bf98df32ad9de))
|
||||
* match leaderboard API nested response schema ([4282eb4](https://github.com/AlexsJones/llmfit/commit/4282eb46331c8d14454baafe1af1760f28e80f48))
|
||||
|
||||
## [0.9.15](https://github.com/AlexsJones/llmfit/compare/v0.9.14...v0.9.15) (2026-04-25)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add DeepSeek V4 model family ([#502](https://github.com/AlexsJones/llmfit/issues/502)) ([0fd800f](https://github.com/AlexsJones/llmfit/commit/0fd800fa1b6bbd7c3280c84b0c384e12ddf55539)), closes [#500](https://github.com/AlexsJones/llmfit/issues/500)
|
||||
* make key 'g' cycle top/bottom ([#454](https://github.com/AlexsJones/llmfit/issues/454)) ([2ccb9bc](https://github.com/AlexsJones/llmfit/commit/2ccb9bc29c71ea333cde4ca99c6dbbf8cd29c6d4))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* MoE min_vram regression — all experts must be in memory ([#497](https://github.com/AlexsJones/llmfit/issues/497)) ([b8f37fd](https://github.com/AlexsJones/llmfit/commit/b8f37fd8131d1cccc23f114634afd1b49cc89cfc))
|
||||
* resolve a build warning for an unused variable ([#331](https://github.com/AlexsJones/llmfit/issues/331)) ([1e68d80](https://github.com/AlexsJones/llmfit/commit/1e68d8081140fa4303fb608180c19b0825f9a760))
|
||||
|
||||
## [0.9.14](https://github.com/AlexsJones/llmfit/compare/v0.9.13...v0.9.14) (2026-04-23)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add Qwen3.6 models and refresh model database with discovery ([5397a8a](https://github.com/AlexsJones/llmfit/commit/5397a8ab43819f9c371588be06c3e2165118d78c))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* MoE GPU-mode uses active params + UD-quant support ([#475](https://github.com/AlexsJones/llmfit/issues/475)) ([2768c0f](https://github.com/AlexsJones/llmfit/commit/2768c0f6dfc20bb90e9289188b7aa468f6c6e369))
|
||||
* populate fit.installed in llmfit fit CLI command ([#491](https://github.com/AlexsJones/llmfit/issues/491)) ([2b67d2e](https://github.com/AlexsJones/llmfit/commit/2b67d2ee5e24a5dc2ad2ff84b2c954a8832d5070))
|
||||
|
||||
## [0.9.13](https://github.com/AlexsJones/llmfit/compare/v0.9.12...v0.9.13) (2026-04-22)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* installed flag always false for models without explicit mapping ([#481](https://github.com/AlexsJones/llmfit/issues/481)) ([#492](https://github.com/AlexsJones/llmfit/issues/492)) ([7d4937f](https://github.com/AlexsJones/llmfit/commit/7d4937fc2564b7c9466fce060e09bd1b66d9a959))
|
||||
|
||||
## [0.9.12](https://github.com/AlexsJones/llmfit/compare/v0.9.11...v0.9.12) (2026-04-22)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --tool-use filter flag to fit command ([#465](https://github.com/AlexsJones/llmfit/issues/465)) ([0ffd8cd](https://github.com/AlexsJones/llmfit/commit/0ffd8cd010e013000c4147d5b99d22418ad41633))
|
||||
* add Filter Popup (F key) with range filters, sort direction, and fit ([#458](https://github.com/AlexsJones/llmfit/issues/458)) ([d182cc3](https://github.com/AlexsJones/llmfit/commit/d182cc34b097dea79887cf30de690d4e39206814))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **ci:** pass crates.io token via env, not argv ([#487](https://github.com/AlexsJones/llmfit/issues/487)) ([7be5edb](https://github.com/AlexsJones/llmfit/commit/7be5edb3448e4a809ebe7a342189d762e09b1af7))
|
||||
* fall back to 127.0.0.1 when localhost Ollama detection fails (fixes [#57](https://github.com/AlexsJones/llmfit/issues/57)) ([#319](https://github.com/AlexsJones/llmfit/issues/319)) ([126f81a](https://github.com/AlexsJones/llmfit/commit/126f81a141237808bb2db7e60bce3db02025f9f5))
|
||||
* parse GPU name from rocm-smi output correctly ([#479](https://github.com/AlexsJones/llmfit/issues/479)) ([fca1984](https://github.com/AlexsJones/llmfit/commit/fca1984bc3bb8c8f4cac9e1965283a5071212aed))
|
||||
* **providers:** prevent argument injection in hf and docker subprocess calls ([#329](https://github.com/AlexsJones/llmfit/issues/329)) ([319c0fe](https://github.com/AlexsJones/llmfit/commit/319c0fe5b2d0845f1279e4440772be17ed8e1de3))
|
||||
* synchronize Cargo.lock and update release workflow to keep it in sync ([#477](https://github.com/AlexsJones/llmfit/issues/477)) ([9a99e69](https://github.com/AlexsJones/llmfit/commit/9a99e699af529c14577b12addd64b4492312c1f8)), closes [#476](https://github.com/AlexsJones/llmfit/issues/476)
|
||||
* **tui:** bind auto-spawned dashboard to loopback by default ([#330](https://github.com/AlexsJones/llmfit/issues/330)) ([6513a69](https://github.com/AlexsJones/llmfit/commit/6513a697d1ba4fb52a0103ff56a5790c631978c5))
|
||||
* **tui:** fix filter popup Esc-revert, cursor alignment, and code cleanup ([5591b90](https://github.com/AlexsJones/llmfit/commit/5591b90946526713e852dec6f5123034051fec7f))
|
||||
* **tui:** use theme background for popups and wrap GGUF source URLs ([#407](https://github.com/AlexsJones/llmfit/issues/407)) ([eb5c9bf](https://github.com/AlexsJones/llmfit/commit/eb5c9bfc5c24f7c38428c830adbd11168d96e1b3)), closes [#344](https://github.com/AlexsJones/llmfit/issues/344)
|
||||
|
||||
## [0.9.11](https://github.com/AlexsJones/llmfit/compare/v0.9.10...v0.9.11) (2026-04-19)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* MoE GPU-mode TPS uses full model size + expert-count-scaled overhead ([#464](https://github.com/AlexsJones/llmfit/issues/464)) ([e1f3245](https://github.com/AlexsJones/llmfit/commit/e1f32459ee6d2f486064cbda045b9a4faaabbc69))
|
||||
* read LM Studio download progress from streaming POST response ([#467](https://github.com/AlexsJones/llmfit/issues/467)) ([3b90e2c](https://github.com/AlexsJones/llmfit/commit/3b90e2cf0f4044d556eb3235b8bd0fbbdc905008)), closes [#463](https://github.com/AlexsJones/llmfit/issues/463)
|
||||
* replace unreachable!() with graceful fallback in estimate_tps_with_gpu ([#459](https://github.com/AlexsJones/llmfit/issues/459)) ([fd1b233](https://github.com/AlexsJones/llmfit/commit/fd1b233de7f52c4d33592a6a2b511da232e5e7d4))
|
||||
|
||||
## [0.9.10](https://github.com/AlexsJones/llmfit/compare/v0.9.9...v0.9.10) (2026-04-18)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --csv flag for spreadsheet-compatible model fit export ([#456](https://github.com/AlexsJones/llmfit/issues/456)) ([48227e6](https://github.com/AlexsJones/llmfit/commit/48227e6032aaa9448d14283f4cab3ba1fb5eb099))
|
||||
* **tui:** add Download Manager view (D) ([c86dbb4](https://github.com/AlexsJones/llmfit/commit/c86dbb491496ff62c3fe2e338fe90a86cba7622e))
|
||||
* **tui:** show VRAM estimate at common context sizes in detail view ([#320](https://github.com/AlexsJones/llmfit/issues/320)) ([3e6f1ab](https://github.com/AlexsJones/llmfit/commit/3e6f1abc0cf5d64520973709d5652075d1e55ac5))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* use Win32_PhysicalMemory to detect AMD APU VRAM on Windows (fixes [#194](https://github.com/AlexsJones/llmfit/issues/194)) ([#452](https://github.com/AlexsJones/llmfit/issues/452)) ([1da9aac](https://github.com/AlexsJones/llmfit/commit/1da9aac210c554a287998bfe8500f790eba4b782))
|
||||
|
||||
## [0.9.9](https://github.com/AlexsJones/llmfit/compare/v0.9.8...v0.9.9) (2026-04-17)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* correct Llama 4 Maverick metadata (expert count, context length) ([#446](https://github.com/AlexsJones/llmfit/issues/446)) ([ac5b115](https://github.com/AlexsJones/llmfit/commit/ac5b115988369de21cdef91b00ea28076dce7c4a))
|
||||
* improve CPU-only TPS accuracy & add cross-platform Advanced Config modal ([#455](https://github.com/AlexsJones/llmfit/issues/455)) ([7e0918b](https://github.com/AlexsJones/llmfit/commit/7e0918b9baeb677f679f13ea754e43d244fe81c3))
|
||||
|
||||
## [0.9.8](https://github.com/AlexsJones/llmfit/compare/v0.9.7...v0.9.8) (2026-04-13)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **ci:** use RELEASE_TOKEN in release-please to trigger downstream workflows ([f216779](https://github.com/AlexsJones/llmfit/commit/f21677927b9d05a348a6ed33da6faaf832f9d82c))
|
||||
* prefer discrete GPU over integrated on hybrid systems ([#303](https://github.com/AlexsJones/llmfit/issues/303)) ([2707b54](https://github.com/AlexsJones/llmfit/commit/2707b54b2ee60ee2132512a4534c1db082701b83))
|
||||
|
||||
## [0.9.7](https://github.com/AlexsJones/llmfit/compare/v0.9.6...v0.9.7) (2026-04-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --force-runtime flag to override automatic runtime selection ([c31099f](https://github.com/AlexsJones/llmfit/commit/c31099f7de99091f3a5c5593a453df89d3feafbc))
|
||||
* add --memory flag to override GPU VRAM autodetection ([9a35a76](https://github.com/AlexsJones/llmfit/commit/9a35a76470b02f51f572fb88e674974f86329b08))
|
||||
* add --memory flag to override GPU VRAM autodetection ([b6e323d](https://github.com/AlexsJones/llmfit/commit/b6e323d72816f8f5920943c573c3897b12ce5b49))
|
||||
* add "Sort by Release Date" to TUI ([f371cd8](https://github.com/AlexsJones/llmfit/commit/f371cd88f6bf2eae61b0fbbc6c3d4b3454ea0fb2))
|
||||
* add "Sort by Release Date" to TUI ([d55670a](https://github.com/AlexsJones/llmfit/commit/d55670aba8ae0f359f8ee0c85444a5bab16d383d))
|
||||
* add 15 popular models from HuggingFace ([eb886d6](https://github.com/AlexsJones/llmfit/commit/eb886d643dcef083650d980ef8c36114310f5110))
|
||||
* Add 15 popular models from HuggingFace (33→48 models) ([e0c0b52](https://github.com/AlexsJones/llmfit/commit/e0c0b528b5e6bed070c2f2f4123f6cddc70b5125))
|
||||
* add 18 new models to database (Feb 2026) ([8612c0e](https://github.com/AlexsJones/llmfit/commit/8612c0e4feaa072eadaa44ca492d4c1e226fe091))
|
||||
* add 18 new models to database (Feb 2026) ([1c040f2](https://github.com/AlexsJones/llmfit/commit/1c040f280fe2a790211c1b6a4d302a3f7edfd00b))
|
||||
* add 18 new models to database (Feb 2026) — conflict-resolved ([0970845](https://github.com/AlexsJones/llmfit/commit/097084568841b5c2a8c7f17f65042612e7e5b68c))
|
||||
* add AWQ/GPTQ support with vLLM inference runtime ([e398490](https://github.com/AlexsJones/llmfit/commit/e3984901f9d5eadff02c3683f8ebd646aededf4c))
|
||||
* add AWQ/GPTQ support with vLLM inference runtime ([d7f2f11](https://github.com/AlexsJones/llmfit/commit/d7f2f11bb29efb56f500669c2e4e820ce6e93d4e))
|
||||
* add Catppuccin color themes to TUI ([f05d299](https://github.com/AlexsJones/llmfit/commit/f05d299f541866721f061ed8599fd9e3072b4f60))
|
||||
* add Catppuccin color themes to TUI ([104b281](https://github.com/AlexsJones/llmfit/commit/104b28130ba246167fb6019d531548090128b189))
|
||||
* add CLI/TUI model diff compare workflow ([3cab068](https://github.com/AlexsJones/llmfit/commit/3cab068f83fda006988631fe4134501db42edde1))
|
||||
* add context-length cap for memory estimation (--max-context) ([0d9a0ef](https://github.com/AlexsJones/llmfit/commit/0d9a0ef8ec7f2d200671ebbb9207b9e1d7bf25a2))
|
||||
* Add Docker containerization support ([42ff7d3](https://github.com/AlexsJones/llmfit/commit/42ff7d35177397c0b95556873b600505e7555032))
|
||||
* Add Docker containerization support ([fbb483c](https://github.com/AlexsJones/llmfit/commit/fbb483cf4c3db735eac8248cd16413dd67c31810))
|
||||
* add Docker Model Runner as a runtime provider ([2a84138](https://github.com/AlexsJones/llmfit/commit/2a84138b001ec02768ebb9afc69274b023c29344))
|
||||
* add fuzzy search with space-separated terms ([89bbe3d](https://github.com/AlexsJones/llmfit/commit/89bbe3df09b40dfee195c5dde4974f36f19d6598))
|
||||
* add fuzzy search with space-separated terms ([3d967f4](https://github.com/AlexsJones/llmfit/commit/3d967f4ec72f3126863dc27b43d49000fd0686fa)), closes [#65](https://github.com/AlexsJones/llmfit/issues/65)
|
||||
* add GGUF download source enrichment for models ([967253b](https://github.com/AlexsJones/llmfit/commit/967253b6098dbb45596999eca5e4a1483d70a8c9))
|
||||
* add Google Gemma 4 models and fix Gemma 3 capabilities ([2cb21c3](https://github.com/AlexsJones/llmfit/commit/2cb21c36b97a64546eb8cc6b49318d42f672a1fb))
|
||||
* add homebrew tap support and update release workflow ([67b6fcf](https://github.com/AlexsJones/llmfit/commit/67b6fcf371255a72538b9aca2912f666b8627978))
|
||||
* add InferenceRuntime enum and MLX quantization support ([186b95e](https://github.com/AlexsJones/llmfit/commit/186b95ef1a76ea3ddb6b6b27d7ae0046ba38fe96))
|
||||
* add license filter for models ([5bb4b1e](https://github.com/AlexsJones/llmfit/commit/5bb4b1e6ea6d9bcd426760090459e301bb38206c))
|
||||
* add license filter for models ([#186](https://github.com/AlexsJones/llmfit/issues/186)) ([33bd708](https://github.com/AlexsJones/llmfit/commit/33bd70870f682d760cdc8d1d3dc7aed9ea2cd978))
|
||||
* add Liquid AI LFM2/LFM2.5 models ([83d038a](https://github.com/AlexsJones/llmfit/commit/83d038a54e7c62d740a4423261959b9c5c49dd33))
|
||||
* add max-context cap for memory estimation ([afba24a](https://github.com/AlexsJones/llmfit/commit/afba24a2a0abb71f8ce7005ed3dcfcccdeb64f0e))
|
||||
* add MiniMax-M2.7 to curated model database ([a5cda9b](https://github.com/AlexsJones/llmfit/commit/a5cda9bbc23e3b494bf67677f7a123601ecff1e3))
|
||||
* add MiniMax-M2.7 to curated model database ([7c16b9d](https://github.com/AlexsJones/llmfit/commit/7c16b9d5ee70b321257d74cb8ca4d5b76cda0c55))
|
||||
* add MlxProvider for MLX model detection and downloads ([7f0ee24](https://github.com/AlexsJones/llmfit/commit/7f0ee2447d1062e5b737ed80777e14d4d207aa5f))
|
||||
* add model comparison workflow (llmfit diff) and TUI compare mode ([76a8a9c](https://github.com/AlexsJones/llmfit/commit/76a8a9c80debad9c2b6df140dc7191778615ff39))
|
||||
* add Ollama mappings for new models ([5c97e75](https://github.com/AlexsJones/llmfit/commit/5c97e75df38a5ba429c3a6ab60dd37b560514d9d))
|
||||
* add Qwen 3.5 series (397B/122B/35B/27B) to model database ([0d550f3](https://github.com/AlexsJones/llmfit/commit/0d550f3ae5415e8e84eb40941fef32cfaa9cd0ab))
|
||||
* add Qwen3-Coder-Next (80B MoE) and Qwen 3.5 Ollama mappings ([5f8a640](https://github.com/AlexsJones/llmfit/commit/5f8a640924e505cc889eb7a4afc70f45c2614474))
|
||||
* add Qwen3.5 small model series (0.8B, 2B, 4B, 9B) ([21cc8ac](https://github.com/AlexsJones/llmfit/commit/21cc8ac2585ac0a5bf4b6ceab99f27c07e51acde))
|
||||
* add Qwen3.5 small model series (0.8B, 2B, 4B, 9B) ([226c5c2](https://github.com/AlexsJones/llmfit/commit/226c5c2d805caa2418992a1a6076f1a2125b6082))
|
||||
* add runtime model database updates from HuggingFace ([d465fb9](https://github.com/AlexsJones/llmfit/commit/d465fb9ddce782ff094b3f462aee71852b02c807))
|
||||
* add runtime model database updates from HuggingFace ([1e20ec3](https://github.com/AlexsJones/llmfit/commit/1e20ec3f13711d7fd7538619fe5347f209c6ec46))
|
||||
* add tensor parallelism awareness for multi-GPU model fitting ([e8a6001](https://github.com/AlexsJones/llmfit/commit/e8a60011d751bffebe65a22ef164a11f8c22a691))
|
||||
* add tensor parallelism awareness for multi-GPU model fitting ([5d99ec2](https://github.com/AlexsJones/llmfit/commit/5d99ec2274ef8e18f2cbf0960bd525f0cf480a05))
|
||||
* add theme switcher with 6 color schemes for TUI ([317c44a](https://github.com/AlexsJones/llmfit/commit/317c44a8b06067938117b4ad7b60e7e3298ab08c))
|
||||
* add theme switcher with 6 color schemes for TUI ([89bd63f](https://github.com/AlexsJones/llmfit/commit/89bd63f0166f7af4c00efd643fe956c58f824f4f))
|
||||
* add tok/s column sorting in TUI ([41b7358](https://github.com/AlexsJones/llmfit/commit/41b73580e7d16c6a318256ae1bd1bc704b457608))
|
||||
* add tok/s sort option to TUI sort cycle and table header highlighting ([82bb526](https://github.com/AlexsJones/llmfit/commit/82bb526b00f19affb7a53edac56f4d4fd73311da))
|
||||
* add use-case popup filter and use-case search matching to TUI ([4b1aeab](https://github.com/AlexsJones/llmfit/commit/4b1aeabb1f3961e4e749714ef6d00b2b41d76a27))
|
||||
* add use-case popup filter to TUI ([4b0f452](https://github.com/AlexsJones/llmfit/commit/4b0f452fa89974aa16a1c311d23890660e1eb74f))
|
||||
* added arc support ([0d5e991](https://github.com/AlexsJones/llmfit/commit/0d5e991a53c3b9ec3639e32318ca74ef70fba5d0))
|
||||
* added in vim like bindings ([a3df62f](https://github.com/AlexsJones/llmfit/commit/a3df62f5654f3e681597266b4f29c7f6c7ac005e))
|
||||
* added LM studio ([70e60cc](https://github.com/AlexsJones/llmfit/commit/70e60ccd03a02d11cdc5aa818a168934530ee44d))
|
||||
* added logo ([86dd737](https://github.com/AlexsJones/llmfit/commit/86dd7379ae62e74df477f70287b24587d144a09c))
|
||||
* added moe ([4723782](https://github.com/AlexsJones/llmfit/commit/47237826f1cd4428a8ebe0fac1a8b5369796882a))
|
||||
* added Rest API ([2adb593](https://github.com/AlexsJones/llmfit/commit/2adb593f44137b479b6fc986b9083d373afad530))
|
||||
* adding ollama as supported provider ([e198c17](https://github.com/AlexsJones/llmfit/commit/e198c17687ee5560384655df52e6bb9dcd5ba014))
|
||||
* adding release please ([c68914a](https://github.com/AlexsJones/llmfit/commit/c68914a5cd51c270585d74c2ffe8fdddc157c34a))
|
||||
* adding serve capabilities ([503229e](https://github.com/AlexsJones/llmfit/commit/503229e5664d7bdd065ff84a30a36dae27a55e90))
|
||||
* **api:** add capabilities, license, supports_tp to model response and new endpoints ([c02e189](https://github.com/AlexsJones/llmfit/commit/c02e189f676a44714bddaff1348495d7863a860d))
|
||||
* append (WSL) to RAM label in tui when running under WSL ([aa30f57](https://github.com/AlexsJones/llmfit/commit/aa30f57276535c84bf835c2b87576644cc69af81))
|
||||
* availability filter [a] in TUI (All / GGUF Avail / Installed) ([9356150](https://github.com/AlexsJones/llmfit/commit/935615088a05b779d0e762603d994bed6f20c246))
|
||||
* bandwidth-based tok/s estimation for known GPUs ([0c552b4](https://github.com/AlexsJones/llmfit/commit/0c552b470316b05976f42c85f7f1472194395a66))
|
||||
* bandwidth-based tok/s estimation for known GPUs ([1c34600](https://github.com/AlexsJones/llmfit/commit/1c34600e9d5388badbdcd5f72e4848ef094e08a8))
|
||||
* cargo fmt ([cde77e3](https://github.com/AlexsJones/llmfit/commit/cde77e35019e1e0e998dcaf2a6fc4b228918a909))
|
||||
* caught some unavailable models on ollama ([1d68bba](https://github.com/AlexsJones/llmfit/commit/1d68bbabf6a0416cf8e619ad38cc56be95f58a32))
|
||||
* caught some unavailable models on ollama ([6aaba9f](https://github.com/AlexsJones/llmfit/commit/6aaba9fd77abda41e6867d4d4018843f875bada7))
|
||||
* **cli:** add --sort for fit output ([8114446](https://github.com/AlexsJones/llmfit/commit/8114446eafbfcad27e3ae097a4a42f4d20aa9e7a))
|
||||
* **cli:** add `--sort` option for fit output ([3a06877](https://github.com/AlexsJones/llmfit/commit/3a06877ce2908bec4c39b5164a1d5a0c908b9d7a))
|
||||
* crate version yank skip rebuild ([b1b0628](https://github.com/AlexsJones/llmfit/commit/b1b06286723f4e21bea81ce7f2602f12ca0fe5bb))
|
||||
* **dashboard:** ship embedded web UI and auto-start from CLI ([96bd53c](https://github.com/AlexsJones/llmfit/commit/96bd53c9c9cee7cbe1d1f1ee08559b6fc4b9aa2e))
|
||||
* **dashboard:** ship embedded web UI and auto-start it from CLI ([37dd0db](https://github.com/AlexsJones/llmfit/commit/37dd0dbca4b9dab1fc96597a64751a4267128fa6))
|
||||
* detect installed Ollama models and support pulling from TUI ([dc05d5a](https://github.com/AlexsJones/llmfit/commit/dc05d5a41222f5961ac544c22e43b7398431c901))
|
||||
* expand mlx-community model mappings ([7a9e917](https://github.com/AlexsJones/llmfit/commit/7a9e9171b10776186f9a259450d0ebfc8d0b2f40))
|
||||
* expand mlx-community model mappings and improve fallback heuristic ([5e8a676](https://github.com/AlexsJones/llmfit/commit/5e8a6769ccf3ae0754b80b91fb39e35144553b74)), closes [#260](https://github.com/AlexsJones/llmfit/issues/260)
|
||||
* fix for qwen3_5moe ([455709f](https://github.com/AlexsJones/llmfit/commit/455709f2fb7524b2c62b455c56454ce3897669c6))
|
||||
* fixed regression in LM studio model list ([cbeccfa](https://github.com/AlexsJones/llmfit/commit/cbeccfa60546b1c150d103621470bf5b2b32bb23))
|
||||
* fixed regression in LM studio model list ([734f556](https://github.com/AlexsJones/llmfit/commit/734f55620114d3b14d17bf24e399b7ace4f7e4c3))
|
||||
* fixed up skill ([38c8350](https://github.com/AlexsJones/llmfit/commit/38c8350e2f78c3207a2c40e1079267db6b707809))
|
||||
* fixed up skill ([368fa47](https://github.com/AlexsJones/llmfit/commit/368fa4728064238c377fb0632c7fc83a5688c504))
|
||||
* fixing vram on apple bug ([bd59767](https://github.com/AlexsJones/llmfit/commit/bd59767f2b6fdc7dfd1352cbb12f7dff17e18d8b))
|
||||
* fixing vram on apple bug ([db666b6](https://github.com/AlexsJones/llmfit/commit/db666b66d4087ecf1b62c1c9f582e765007eb30b))
|
||||
* fixing vram on apple bug ([c26d13b](https://github.com/AlexsJones/llmfit/commit/c26d13bde0990549c6bc1c3f090d1935c23ec036))
|
||||
* fixing vram on apple bug ([085cd25](https://github.com/AlexsJones/llmfit/commit/085cd25a3730c762ed21e9e0c4570ecb6a17323c))
|
||||
* GGUF download source enrichment (unsloth & bartowski) ([6e3c828](https://github.com/AlexsJones/llmfit/commit/6e3c828123009f76088265a11143e8d5e7474ed0))
|
||||
* hardware simulation mode and CLI hardware overrides ([002eeff](https://github.com/AlexsJones/llmfit/commit/002eeff6acf61970c2d3bf471a3b2a1afc3f82ba)), closes [#322](https://github.com/AlexsJones/llmfit/issues/322)
|
||||
* **hardware:** detect NVIDIA Tegra/Grace Blackwell unified memory via ATS (rebased [#93](https://github.com/AlexsJones/llmfit/issues/93)) ([b454697](https://github.com/AlexsJones/llmfit/commit/b45469797c502d76e2b904ccf50cdb7d5781a648))
|
||||
* **hardware:** detect NVIDIA Tegra/Grace Blackwell unified memory via ATS addressing mode ([dd1caa6](https://github.com/AlexsJones/llmfit/commit/dd1caa60d4ffec36605f7fed9c2cb77ed3b1a23e))
|
||||
* improvements based on [#12](https://github.com/AlexsJones/llmfit/issues/12) ([7997525](https://github.com/AlexsJones/llmfit/commit/7997525fc248e3cff845b47a14afac9694401cd0))
|
||||
* increased model count ([5be634f](https://github.com/AlexsJones/llmfit/commit/5be634f3455f68db82aedc21357df243b60c8688))
|
||||
* increment version ([d88bd3f](https://github.com/AlexsJones/llmfit/commit/d88bd3fecd7e11cca97c6426fa4aee9ec18df760))
|
||||
* installed status for lm studio ([a1cad30](https://github.com/AlexsJones/llmfit/commit/a1cad30d1cf6cf08f70833347487c480d5650350))
|
||||
* mlx disablement on non-apple hw ([c26bb49](https://github.com/AlexsJones/llmfit/commit/c26bb4927ab54dde78fab827125c5901ea6f238f))
|
||||
* MLX-native inference support for Apple Silicon ([993599e](https://github.com/AlexsJones/llmfit/commit/993599e873141a6676ce5ce428cec15fb5201c46))
|
||||
* model detail modal + Ollama download in desktop app ([c92053f](https://github.com/AlexsJones/llmfit/commit/c92053f38e6980a6c0865d0637a49deeb9c0ce52))
|
||||
* model detail modal + Ollama download in desktop app ([dacae77](https://github.com/AlexsJones/llmfit/commit/dacae77a01739ef50084e3ab6847b7bf5850f290))
|
||||
* overall to the scoring system ([e689f95](https://github.com/AlexsJones/llmfit/commit/e689f95e90bda56a417067cfe7033c7b0b865ad1))
|
||||
* overall to the scoring system ([d411a8d](https://github.com/AlexsJones/llmfit/commit/d411a8dd773fa2fdd813f3c5026074e76b85b51e))
|
||||
* overall to the scoring system ([5ca2467](https://github.com/AlexsJones/llmfit/commit/5ca24672985c7a1dfc02334a537dce6f4c8502c4))
|
||||
* persist filter state across sessions ([#430](https://github.com/AlexsJones/llmfit/issues/430)) ([c3962e4](https://github.com/AlexsJones/llmfit/commit/c3962e4645ccd11bb6a5e5cf23a222e30c98af93))
|
||||
* persist theme selection to ~/.config/llmfit/theme ([765c87b](https://github.com/AlexsJones/llmfit/commit/765c87b4ea8352855fc3774f8af06dee70ae63ed))
|
||||
* **plan:** KV cache fidelity + KV quant flag + TurboQuant gating ([1f1b6fc](https://github.com/AlexsJones/llmfit/commit/1f1b6fc25a04ee1cd7881f4e33becf7d24b95588))
|
||||
* plumbing 2 ([209f9d4](https://github.com/AlexsJones/llmfit/commit/209f9d4b1814923b88bf928d78fdc193fb37af9b))
|
||||
* plumbing 2 ([ef18920](https://github.com/AlexsJones/llmfit/commit/ef1892077c9bbd66bf47b20153ff6f5afff688fa))
|
||||
* pull functionality ([e193d56](https://github.com/AlexsJones/llmfit/commit/e193d5678f954cc56c6a7d1aefad1616b29d5607))
|
||||
* rebased ([ac6639d](https://github.com/AlexsJones/llmfit/commit/ac6639d1b44e8f100fadd2044d571c66c9988d06))
|
||||
* release plumbing ([648a8c1](https://github.com/AlexsJones/llmfit/commit/648a8c179cf174f69a207bc7683e5f6be1a037da))
|
||||
* release plumbing ([8215ba8](https://github.com/AlexsJones/llmfit/commit/8215ba80304b77765faac85dba3525434f2f3b9b))
|
||||
* removed exact model count as it increases so often ([1a19c42](https://github.com/AlexsJones/llmfit/commit/1a19c4258f9435b2d3eddaaade7fd008b49919eb))
|
||||
* reworked available models for download ([6d56b03](https://github.com/AlexsJones/llmfit/commit/6d56b03c128ba720787e9562270dae6c722fae2c))
|
||||
* show approximate disk space usage ([#134](https://github.com/AlexsJones/llmfit/issues/134)) ([#304](https://github.com/AlexsJones/llmfit/issues/304)) ([32c835b](https://github.com/AlexsJones/llmfit/commit/32c835bf421a6b0727b32e5c2cddba8bf5c581e7))
|
||||
* split-pane TUI detail view for GGUF downloads ([de5c53b](https://github.com/AlexsJones/llmfit/commit/de5c53bd6426d51292421460923ea4a478f2da94))
|
||||
* support for windows vulkan ([5a76a8f](https://github.com/AlexsJones/llmfit/commit/5a76a8f956e4732fe4fe9a12020d34f476aa9f0b))
|
||||
* supporting 94 models ([c68d7aa](https://github.com/AlexsJones/llmfit/commit/c68d7aad0c0a00cbf12ae49c7325d74bf0142fce))
|
||||
* surface runtime info in TUI, CLI, and JSON output ([4bdb4c4](https://github.com/AlexsJones/llmfit/commit/4bdb4c487eeb4e2fdd1943bfe21c38081a1a1091))
|
||||
* This PR is to add the ability for capabilities to be described ([d09b8fe](https://github.com/AlexsJones/llmfit/commit/d09b8febedd0a00e1ddf10fe72db2ab8ffa07210))
|
||||
* This PR is to add the ability for capabilities to be described ([05768af](https://github.com/AlexsJones/llmfit/commit/05768af09db6323c38f2ac5b27a6214a8133caf1))
|
||||
* TUI GGUF downloads section, default enrichment, caching ([e7dc0b2](https://github.com/AlexsJones/llmfit/commit/e7dc0b216cc2c22af1478fd91cfd0a711009ad9c))
|
||||
* **tui:** add runtime/backend filter and help popup (v0.9.2) ([ad28138](https://github.com/AlexsJones/llmfit/commit/ad28138aa1362f74caae9c6e3abaf49ba68b7480))
|
||||
* update key bindings and add Catppuccin themes to README files ([c41567b](https://github.com/AlexsJones/llmfit/commit/c41567b5caa7bf6ad49a27bd5e4e895861038e74))
|
||||
* updated build actions ([59a8af4](https://github.com/AlexsJones/llmfit/commit/59a8af4ced8492da2fbcd954ec4b300ba40cf954))
|
||||
* updated demo ([6c32985](https://github.com/AlexsJones/llmfit/commit/6c32985afc222c000a5644ffaaf4e92d16612900))
|
||||
* updated demo ([42fbeeb](https://github.com/AlexsJones/llmfit/commit/42fbeeb114753045ab10af8e1e6c9d0d201dd1ef))
|
||||
* updated images ([20205bd](https://github.com/AlexsJones/llmfit/commit/20205bd4a715f3c65337836fbb507774fae47123))
|
||||
* updated models ([dfb9afb](https://github.com/AlexsJones/llmfit/commit/dfb9afb743d7af348e86f50892584822d85b3ce5))
|
||||
* updated sorting and new models ([744465d](https://github.com/AlexsJones/llmfit/commit/744465d2a88b04ea737feec75dd000d9347cf9ab))
|
||||
* updated tui to support multiple providers better and also multiple GPU suppor ([df338e8](https://github.com/AlexsJones/llmfit/commit/df338e821b7b411b30b6be87b6a4202ced68dabb))
|
||||
* updated urls ([ce14772](https://github.com/AlexsJones/llmfit/commit/ce14772ea18da3d7b0d6554ed678295fd97d5480))
|
||||
* updated version ([4d1542b](https://github.com/AlexsJones/llmfit/commit/4d1542b582d0af0a7bdb22f9d89573cc703d5f36))
|
||||
* updating models ([d4f0a0e](https://github.com/AlexsJones/llmfit/commit/d4f0a0e19a880d95e500bf524bf52c39b56c0bf1))
|
||||
* version bump llama.cpp added ([1643214](https://github.com/AlexsJones/llmfit/commit/1643214e6547e8ea98a84f56101079ed6c737b18))
|
||||
* version bump llama.cpp added ([319fb90](https://github.com/AlexsJones/llmfit/commit/319fb9065f88480d12ba9a3d15c3d323c1f448d4))
|
||||
* **web:** add 10 color themes from TUI theme.rs ([ab14a06](https://github.com/AlexsJones/llmfit/commit/ab14a0699e6765b8cb5c03a4a16a105064c296c7))
|
||||
* **web:** add side-by-side model comparison view ([ccb56e9](https://github.com/AlexsJones/llmfit/commit/ccb56e964b875c8db31946d5021bd646dc194ac6))
|
||||
* **web:** replace freeform limit input with preset options ([96981ec](https://github.com/AlexsJones/llmfit/commit/96981ec65e2642dd8acc91ff7d512d2b4cb3b90f))
|
||||
* **web:** split App.jsx into Header, SystemPanel, FilterBar, ModelTable, DetailPanel ([950e620](https://github.com/AlexsJones/llmfit/commit/950e6208bfd14673820f5acbeb7ed193f9af7a1d))
|
||||
* working on v0.8.1 ([696107d](https://github.com/AlexsJones/llmfit/commit/696107d7650fd6351dc53c7f00bef7c6065a1172))
|
||||
* workspace restructure + Tauri desktop app ([83ded27](https://github.com/AlexsJones/llmfit/commit/83ded27beac5b1772dc9aeec7638d44a4b6e5fb1))
|
||||
* workspace restructure + Tauri desktop app ([3fb10d0](https://github.com/AlexsJones/llmfit/commit/3fb10d019f828662a21619d282570cf32f0ff833))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* accept OLLAMA_HOST values without URL scheme ([#166](https://github.com/AlexsJones/llmfit/issues/166)) ([c647b13](https://github.com/AlexsJones/llmfit/commit/c647b13787edd4031713196d42d42543139bae51))
|
||||
* add --local flag to install script and improve fallback logic ([1e5cda1](https://github.com/AlexsJones/llmfit/commit/1e5cda1a0d4bd7ca2bf50abd553599f57efdbb78))
|
||||
* add --local flag to install script and improve fallback logic ([23b500e](https://github.com/AlexsJones/llmfit/commit/23b500e8bccd25b0bf7ee61a5181147f67e4ca39)), closes [#56](https://github.com/AlexsJones/llmfit/issues/56)
|
||||
* add AMD RX 9060 series to VRAM estimation database ([1c02da8](https://github.com/AlexsJones/llmfit/commit/1c02da8b74af828d7f1386a261073063647efd88))
|
||||
* add AMD RX 9060 series to VRAM estimation database ([9305810](https://github.com/AlexsJones/llmfit/commit/9305810ad7f226abb8f2513e04c45250f827b8e6)), closes [#55](https://github.com/AlexsJones/llmfit/issues/55)
|
||||
* add download confirmation to prevent accidental pulls ([9e4e881](https://github.com/AlexsJones/llmfit/commit/9e4e881c78fae4e9bd5f5adbdb9dfbf8302ed56c))
|
||||
* add download confirmation to prevent accidental pulls ([#95](https://github.com/AlexsJones/llmfit/issues/95)) ([5b56d74](https://github.com/AlexsJones/llmfit/commit/5b56d74e422130718e635e4d82533d8e34f7670d))
|
||||
* address AlexsJones review comments ([587252b](https://github.com/AlexsJones/llmfit/commit/587252b0124e3e7f8dbcdce4ac1ccaed7c7345de))
|
||||
* address Copilot review comments ([74384c9](https://github.com/AlexsJones/llmfit/commit/74384c9ea7d1215aa38c455b20d5005bbc8fa361))
|
||||
* AMD Ryzen AI unified memory detection and --memory override ([174fc10](https://github.com/AlexsJones/llmfit/commit/174fc10bb93cccdfdfd1441cc9bc60d1682d1088))
|
||||
* AMD unified memory detection and --memory override ([c763717](https://github.com/AlexsJones/llmfit/commit/c76371785ef93761eb411b67acc3202832ab9ce5)), closes [#89](https://github.com/AlexsJones/llmfit/issues/89) [#91](https://github.com/AlexsJones/llmfit/issues/91)
|
||||
* bump Dockerfile Rust version to 1.88 for dependency compatibility ([c25c02e](https://github.com/AlexsJones/llmfit/commit/c25c02e00b2709df869fff8fd72c9419303d504a))
|
||||
* bump Dockerfile Rust version to 1.88 for dependency compatibility ([6304ba0](https://github.com/AlexsJones/llmfit/commit/6304ba0c19f804353afc4c6e79c29cee8a51ae70))
|
||||
* cap default estimation context at 8192 tokens ([#311](https://github.com/AlexsJones/llmfit/issues/311)) ([4699ecb](https://github.com/AlexsJones/llmfit/commit/4699ecbf787a89ae4fbb246896b0c9711f893a12))
|
||||
* **ci:** switch release-please to simple type for workspace version inheritance ([48be41a](https://github.com/AlexsJones/llmfit/commit/48be41aad22582a5921f0e1b3db2918aaad92950))
|
||||
* **cli:** default auto dashboard host to 0.0.0.0 ([4553fbf](https://github.com/AlexsJones/llmfit/commit/4553fbfa2a00b36626b3feef49b2fbfe9d888ae0))
|
||||
* **cli:** make dashboard auto-start controllable and self-cleaning ([0c51ad6](https://github.com/AlexsJones/llmfit/commit/0c51ad6ea192da24641aade2c7d22aec0db0d5db))
|
||||
* correct crates.io metadata and prepare for publishing ([f8b08a0](https://github.com/AlexsJones/llmfit/commit/f8b08a0dd6f433bf9bdc7d93288ff653f263f03d))
|
||||
* correct crates.io metadata and prepare for publishing ([2ce5858](https://github.com/AlexsJones/llmfit/commit/2ce5858ca4ae4514f95204c8b696d8173c5e9415)), closes [#58](https://github.com/AlexsJones/llmfit/issues/58)
|
||||
* correct PCI ID reading on Linux and improve AMD GPU identification ([dcfde57](https://github.com/AlexsJones/llmfit/commit/dcfde579e5d3c3df6da3d2f7e152cf13e6a06fb8))
|
||||
* correctly estimate VRAM for APU integrated GPUs ([2df3298](https://github.com/AlexsJones/llmfit/commit/2df329823597b057f3151ead64bbee38be393916))
|
||||
* correctly estimate VRAM for APU integrated GPUs (Radeon Graphics) ([5797f10](https://github.com/AlexsJones/llmfit/commit/5797f10dbc43dafb0346be4d411c91b16025a5a2)), closes [#25](https://github.com/AlexsJones/llmfit/issues/25)
|
||||
* Default theme uses terminal colors for light/dark compat ([72a6d8d](https://github.com/AlexsJones/llmfit/commit/72a6d8d7678ac90c7f3fafeb6a3cbd1ab25bb830)), closes [#67](https://github.com/AlexsJones/llmfit/issues/67)
|
||||
* **desktop:** prevent XSS via inline onclick handler in modal ([#323](https://github.com/AlexsJones/llmfit/issues/323)) ([d20dbee](https://github.com/AlexsJones/llmfit/commit/d20dbeec9f8c3e7378aa2a386d684edda9432523))
|
||||
* detect installed Ollama tag variants with suffixes ([#165](https://github.com/AlexsJones/llmfit/issues/165)) ([7092fa6](https://github.com/AlexsJones/llmfit/commit/7092fa63036a4cb04e5d0315665968f85268ac05))
|
||||
* detect NVIDIA GB10/GB20 as unified memory GPUs ([a77cf36](https://github.com/AlexsJones/llmfit/commit/a77cf3631aa21339e215418cdeb9a8fab90891a1))
|
||||
* detect NVIDIA GB10/GB20 as unified memory GPUs ([#83](https://github.com/AlexsJones/llmfit/issues/83), [#17](https://github.com/AlexsJones/llmfit/issues/17)) ([c0a584a](https://github.com/AlexsJones/llmfit/commit/c0a584a13d14bf71d03d12cc6f0bcd2c71f3d044))
|
||||
* discover GGUF files in HuggingFace subdirectories ([#291](https://github.com/AlexsJones/llmfit/issues/291)) ([4e8a596](https://github.com/AlexsJones/llmfit/commit/4e8a5964d731f8ea39dcc25a474970b83f48b966))
|
||||
* docker action version ([7a38a25](https://github.com/AlexsJones/llmfit/commit/7a38a25423d3841d3fec200815e55f1afc56e62d))
|
||||
* docker action version pin ([da96503](https://github.com/AlexsJones/llmfit/commit/da965030a022c422232560259a053ec3d54ab620))
|
||||
* **download:** fetch all shards of multi-part GGUF models ([b7494d9](https://github.com/AlexsJones/llmfit/commit/b7494d9910dff9e515337a6af9fc2acad59a2628))
|
||||
* filter AWQ/GPTQ models by GPU compute capability ([e720a0e](https://github.com/AlexsJones/llmfit/commit/e720a0e9ebf43a1a2ec8630869c15dc9b827084e)), closes [#257](https://github.com/AlexsJones/llmfit/issues/257)
|
||||
* find Tauri bundle in correct target directory ([19c0ac3](https://github.com/AlexsJones/llmfit/commit/19c0ac315f9376fbb9b2ff882720d6971c405951))
|
||||
* **fmt:** align providers formatting and enrich TUI compare metrics ([a65e4c5](https://github.com/AlexsJones/llmfit/commit/a65e4c53a38f5a582a259801dcdc47ab7cf15f7a))
|
||||
* hide MLX models on non-Apple-Silicon hardware ([#113](https://github.com/AlexsJones/llmfit/issues/113)) ([1e3cf8a](https://github.com/AlexsJones/llmfit/commit/1e3cf8a6a25f3f2e84a29a5becd01317e04a6e4b))
|
||||
* hide MLX models on non-Apple-Silicon hardware ([#113](https://github.com/AlexsJones/llmfit/issues/113)) ([1052819](https://github.com/AlexsJones/llmfit/commit/1052819f40c8cc96efa1d46429f2e09a73c5e1fd))
|
||||
* hide MLX-only models on non-Apple Silicon systems ([ab54fa9](https://github.com/AlexsJones/llmfit/commit/ab54fa927b79c0335e931a1989fa67b15eb1c8a3))
|
||||
* ignore GGUF tests that require network access ([#306](https://github.com/AlexsJones/llmfit/issues/306)) ([8fd8f4d](https://github.com/AlexsJones/llmfit/commit/8fd8f4d9acc27506f78117029f3b540e6028ee6b))
|
||||
* iGPU inflating GPU count and force-runtime being ignored ([#271](https://github.com/AlexsJones/llmfit/issues/271)) ([f584d7e](https://github.com/AlexsJones/llmfit/commit/f584d7edf7c42836441e3dcd0d5e911ca2bc4a2e))
|
||||
* improve Android CPU and Vulkan GPU detection ([0ebb92a](https://github.com/AlexsJones/llmfit/commit/0ebb92a42960cdf4e5154616d5b1925d27f34a3b))
|
||||
* improve download error messages for incompatible model formats ([0e2f9cc](https://github.com/AlexsJones/llmfit/commit/0e2f9ccc42a4ea3954e0320bce16d3500306fa46)), closes [#123](https://github.com/AlexsJones/llmfit/issues/123) [#198](https://github.com/AlexsJones/llmfit/issues/198)
|
||||
* improve MoE tok/s estimate and clarify baseline speed labels ([81f88c4](https://github.com/AlexsJones/llmfit/commit/81f88c4ee78ab686fc47cefa72115025ca9e0154))
|
||||
* incorrect installed model count in TUI status bar ([307c74f](https://github.com/AlexsJones/llmfit/commit/307c74f265b6e55f3c803eedf0a2e90bf8a499f3))
|
||||
* invoke hf instead of huggingface-cli ([752680f](https://github.com/AlexsJones/llmfit/commit/752680f740540ae5f6e1c40a1f924fd6197a6634))
|
||||
* invoke hf instead of huggingface-cli ([78f8268](https://github.com/AlexsJones/llmfit/commit/78f8268070f58c1c2647cfc0ceffa7ffab8a2e1e))
|
||||
* pr review fixes ([25952da](https://github.com/AlexsJones/llmfit/commit/25952da6a6f707ad16fac33bdea0c17d38c5c064))
|
||||
* prefer discrete GPU over integrated on Windows ([#303](https://github.com/AlexsJones/llmfit/issues/303)) ([3312a40](https://github.com/AlexsJones/llmfit/commit/3312a407901392806a623e653fa2b9bd20155b2a))
|
||||
* prefer exact matches in info selection ([68272d4](https://github.com/AlexsJones/llmfit/commit/68272d40aeea33c482cc5e211902339ae6b72009))
|
||||
* prefer exact matches in info selection ([883ea1f](https://github.com/AlexsJones/llmfit/commit/883ea1f90901f5020b655691588cdc70329c2adc))
|
||||
* query local providers in `recommend` CLI command ([2477d1d](https://github.com/AlexsJones/llmfit/commit/2477d1dfa285a2eb3a46209c1f5487eaca20aeb8))
|
||||
* regression in json only mode ([36e8503](https://github.com/AlexsJones/llmfit/commit/36e8503e6c21e40a11185fba0c84badd53f039a8))
|
||||
* remove unused function to eliminate build-time warning ([4406582](https://github.com/AlexsJones/llmfit/commit/44065822115469d52bbd1afb5b8fe44f73c8756b))
|
||||
* remove unused function to eliminate build-time warning ([289c8b8](https://github.com/AlexsJones/llmfit/commit/289c8b8fc6c9872f4c2ae20373e5c42c23ab0aa8))
|
||||
* Runtimes are installed but no downloadable artifact exist ([9424204](https://github.com/AlexsJones/llmfit/commit/9424204e7364586889c0714b316433e3b5f63ea0))
|
||||
* support owner-scoped MLX pulls and robust tag normalization ([7a9e476](https://github.com/AlexsJones/llmfit/commit/7a9e476e3b5b7926ccc7ee671aa538fffd5f05c8))
|
||||
* support owner-scoped MLX repos and normalize MLX pull tags ([f0e516c](https://github.com/AlexsJones/llmfit/commit/f0e516c5c474228d86eb2bf86ff1caa1d5668f41))
|
||||
* support sudo in piped install script ([9fa1960](https://github.com/AlexsJones/llmfit/commit/9fa19609064bb93585b2cf7a8af04ca07cb83de7))
|
||||
* support sudo in piped install script ([cf61557](https://github.com/AlexsJones/llmfit/commit/cf615573acd329a6b4e525d4085b5cab301680f3)), closes [#158](https://github.com/AlexsJones/llmfit/issues/158)
|
||||
* surface MoE offloaded RAM in JSON output ([96b73ff](https://github.com/AlexsJones/llmfit/commit/96b73ff7bc85b98ece85ea61f3c0d8acc5dd3559))
|
||||
* surface MoE offloaded RAM in JSON output ([929eaad](https://github.com/AlexsJones/llmfit/commit/929eaad2aae7be9a8dd1fe962b5617fb10b0142e)), closes [#230](https://github.com/AlexsJones/llmfit/issues/230)
|
||||
* **tui:** avoid borrow conflict when marking compare model ([ae60aa5](https://github.com/AlexsJones/llmfit/commit/ae60aa56ab0f4f9b039d309686c3d7ae714ea6f9))
|
||||
* **tui:** write dashboard pid file under ~/.llmfit, not /tmp ([#324](https://github.com/AlexsJones/llmfit/issues/324)) ([00967f1](https://github.com/AlexsJones/llmfit/commit/00967f128d2606b84ba33e056c42ce78d265ef48))
|
||||
* typo in CHANGELOG.md (suppor -> support) ([1e26ec2](https://github.com/AlexsJones/llmfit/commit/1e26ec22f4d0bcbe071312c453fb01c5df8db926))
|
||||
* typo in CHANGELOG.md (suppor -> support) ([aa6cd00](https://github.com/AlexsJones/llmfit/commit/aa6cd00d8241b6391b575af408a1070db89dda66))
|
||||
* update ([2b6767d](https://github.com/AlexsJones/llmfit/commit/2b6767dc15e8b40b52251c3a42e3f4a73694e2b0))
|
||||
* update OpenClaw skill to match actual CLI output ([df585a7](https://github.com/AlexsJones/llmfit/commit/df585a759c8f24942f5ddfa2a15819cd7b35c540))
|
||||
* update OpenClaw skill to match actual CLI output ([d53c192](https://github.com/AlexsJones/llmfit/commit/d53c1926a65f20baf9571e4ff73ae7d4207961fe))
|
||||
* use accurate model counts instead of len()/2 for installed display ([7a273cc](https://github.com/AlexsJones/llmfit/commit/7a273cc3b42629106c818211d76ded1b2b06f316)), closes [#189](https://github.com/AlexsJones/llmfit/issues/189)
|
||||
* use aggregated VRAM for multi-GPU fit scoring ([dc3db39](https://github.com/AlexsJones/llmfit/commit/dc3db39dafb62b7a41c045928ec0e08dd17ab66b))
|
||||
* use aggregated VRAM for multi-GPU fit scoring ([f55882d](https://github.com/AlexsJones/llmfit/commit/f55882d9f664df98c6058fe7662cb95a20db35e9)), closes [#68](https://github.com/AlexsJones/llmfit/issues/68)
|
||||
* use MoE active parameters for tok/s estimation and clarify baseline speed labels ([47e70a1](https://github.com/AlexsJones/llmfit/commit/47e70a17433fbd79b8a93be84a429b918dfd6661))
|
||||
* use per-card VRAM instead of summed for multi-GPU systems ([2f1f9f4](https://github.com/AlexsJones/llmfit/commit/2f1f9f4e0ddac364e0a1ba4894adf9a6161926fc))
|
||||
* use per-card VRAM instead of summed for multi-GPU systems ([f80b4b0](https://github.com/AlexsJones/llmfit/commit/f80b4b049173fa818019dc657b4bd8849d87e2d6))
|
||||
* use quantization in path selection for both dense and MoE models ([#49](https://github.com/AlexsJones/llmfit/issues/49)) ([8703d43](https://github.com/AlexsJones/llmfit/commit/8703d431b971d95071cec40c66d514df7c995249))
|
||||
* **web:** align dashboard hero with project tagline ([d4e77c1](https://github.com/AlexsJones/llmfit/commit/d4e77c167e835c9123b89c340b7c0d3579e2d26f))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* **api:** optimize embedded asset serving and cache policy ([4d8c0cc](https://github.com/AlexsJones/llmfit/commit/4d8c0cc08384bcc1d3e50dc379014f8b3a31bf56))
|
||||
|
||||
## [0.9.6](https://github.com/AlexsJones/llmfit/compare/v0.9.5...v0.9.6) (2026-04-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* persist filter state across sessions ([#430](https://github.com/AlexsJones/llmfit/issues/430)) ([c3962e4](https://github.com/AlexsJones/llmfit/commit/c3962e4645ccd11bb6a5e5cf23a222e30c98af93))
|
||||
|
||||
## [0.9.5](https://github.com/AlexsJones/llmfit/compare/v0.9.4...v0.9.5) (2026-04-11)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **plan:** KV cache fidelity + KV quant flag + TurboQuant gating ([1f1b6fc](https://github.com/AlexsJones/llmfit/commit/1f1b6fc25a04ee1cd7881f4e33becf7d24b95588))
|
||||
|
||||
## [0.9.4](https://github.com/AlexsJones/llmfit/compare/v0.9.3...v0.9.4) (2026-04-11)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **download:** fetch all shards of multi-part GGUF models ([b7494d9](https://github.com/AlexsJones/llmfit/commit/b7494d9910dff9e515337a6af9fc2acad59a2628))
|
||||
|
||||
## [0.9.3](https://github.com/AlexsJones/llmfit/compare/v0.9.2...v0.9.3) (2026-04-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* hardware simulation mode and CLI hardware overrides ([002eeff](https://github.com/AlexsJones/llmfit/commit/002eeff6acf61970c2d3bf471a3b2a1afc3f82ba)), closes [#322](https://github.com/AlexsJones/llmfit/issues/322)
|
||||
* show approximate disk space usage ([#134](https://github.com/AlexsJones/llmfit/issues/134)) ([#304](https://github.com/AlexsJones/llmfit/issues/304)) ([32c835b](https://github.com/AlexsJones/llmfit/commit/32c835bf421a6b0727b32e5c2cddba8bf5c581e7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **ci:** switch release-please to simple type for workspace version inheritance ([48be41a](https://github.com/AlexsJones/llmfit/commit/48be41aad22582a5921f0e1b3db2918aaad92950))
|
||||
* **desktop:** prevent XSS via inline onclick handler in modal ([#323](https://github.com/AlexsJones/llmfit/issues/323)) ([d20dbee](https://github.com/AlexsJones/llmfit/commit/d20dbeec9f8c3e7378aa2a386d684edda9432523))
|
||||
* **tui:** write dashboard pid file under ~/.llmfit, not /tmp ([#324](https://github.com/AlexsJones/llmfit/issues/324)) ([00967f1](https://github.com/AlexsJones/llmfit/commit/00967f128d2606b84ba33e056c42ce78d265ef48))
|
||||
|
||||
## [0.3.7](https://github.com/AlexsJones/llmfit/compare/v0.3.6...v0.3.7) (2026-02-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --memory flag to override GPU VRAM autodetection ([9a02f6e](https://github.com/AlexsJones/llmfit/commit/9a02f6e1616f59783ccff5b007c25213854f63b9))
|
||||
* add --memory flag to override GPU VRAM autodetection ([39c5486](https://github.com/AlexsJones/llmfit/commit/39c5486aa3d94f9b9ef36e29642b64d848d0d2b0))
|
||||
* add 15 popular models from HuggingFace ([128a020](https://github.com/AlexsJones/llmfit/commit/128a020323897a67ed5d12dd397bcf4924a6bf51))
|
||||
* Add 15 popular models from HuggingFace (33→48 models) ([c45606b](https://github.com/AlexsJones/llmfit/commit/c45606bdb235b6bfe616bb616b1364a97e76f0c1))
|
||||
* add homebrew tap support and update release workflow ([db09473](https://github.com/AlexsJones/llmfit/commit/db094734288d17a49d9c3c5c99859fe0d7dc976d))
|
||||
* added arc support ([b5892fc](https://github.com/AlexsJones/llmfit/commit/b5892fc2ff313e71f57b7d793c7444d2aaadc0bd))
|
||||
* added logo ([c21d416](https://github.com/AlexsJones/llmfit/commit/c21d4168f2bcd6da878848f9a6f97179d558606b))
|
||||
* added moe ([ac7ffe4](https://github.com/AlexsJones/llmfit/commit/ac7ffe4ed79eb22ec43cf7bc20e8cd8d102d16a9))
|
||||
* adding release please ([f2bfc7f](https://github.com/AlexsJones/llmfit/commit/f2bfc7fcf2587b74e05d8ad9d1041be6de456e69))
|
||||
* append (WSL) to RAM label in tui when running under WSL ([e0397cf](https://github.com/AlexsJones/llmfit/commit/e0397cf51025b393b0d4024c4ae67200ee206390))
|
||||
* caught some unavailable models on ollama ([b9f38da](https://github.com/AlexsJones/llmfit/commit/b9f38da9579040a7c2bada55838c5541474883ca))
|
||||
* caught some unavailable models on ollama ([c0f7c20](https://github.com/AlexsJones/llmfit/commit/c0f7c20f61cdd9ae692de6ca66344befba2fafa9))
|
||||
* detect installed Ollama models and support pulling from TUI ([4159aaf](https://github.com/AlexsJones/llmfit/commit/4159aaf304b3b421679f8231cf574465783d5b41))
|
||||
* first pass ([855ad3d](https://github.com/AlexsJones/llmfit/commit/855ad3d34160cce6200c0ff128c34bcdcb0b922b))
|
||||
* fixed up skill ([fcb712a](https://github.com/AlexsJones/llmfit/commit/fcb712a98ac785ad83ad689d5300f17cb80a3f1c))
|
||||
* fixed up skill ([1f7d1de](https://github.com/AlexsJones/llmfit/commit/1f7d1de547a31202b9d34dd62bf543f5a22b2de7))
|
||||
* fixing vram on apple bug ([5e08754](https://github.com/AlexsJones/llmfit/commit/5e087549c7c1523f4d5df72bd8a915330498a795))
|
||||
* fixing vram on apple bug ([b3deca1](https://github.com/AlexsJones/llmfit/commit/b3deca1d9eac16283d0e9269c68a1af1dfc871ab))
|
||||
* fixing vram on apple bug ([92ddb0e](https://github.com/AlexsJones/llmfit/commit/92ddb0e82579c6018d1acb4e3dfbe1df7d582605))
|
||||
* fixing vram on apple bug ([42b2081](https://github.com/AlexsJones/llmfit/commit/42b2081577bed23176c0f87d1ad0b142cce23872))
|
||||
* improvements based on [#12](https://github.com/AlexsJones/llmfit/issues/12) ([5428ef8](https://github.com/AlexsJones/llmfit/commit/5428ef8cdd42e88bced1459b55b480aab767637c))
|
||||
* increased model count ([156b29d](https://github.com/AlexsJones/llmfit/commit/156b29deb077a1d66948254b370597a118fd5daf))
|
||||
* increment version ([283bebb](https://github.com/AlexsJones/llmfit/commit/283bebb8eca5da2fc7124b665ae773fda48aed93))
|
||||
* overall to the scoring system ([f475938](https://github.com/AlexsJones/llmfit/commit/f4759381d23b834e0a42a4699d23fb3f858fe677))
|
||||
* overall to the scoring system ([b0696cf](https://github.com/AlexsJones/llmfit/commit/b0696cf297f1cb11247493355406d8b9c56510db))
|
||||
* overall to the scoring system ([37e2e10](https://github.com/AlexsJones/llmfit/commit/37e2e10076f450f79165d92541baf04957ec2fe9))
|
||||
* plumbing 2 ([1c615bb](https://github.com/AlexsJones/llmfit/commit/1c615bb57b7395f9be888245f8157dec2bab8bb4))
|
||||
* plumbing 2 ([dd6a3ec](https://github.com/AlexsJones/llmfit/commit/dd6a3ec20e09ae72eada1fada73a6392c9673221))
|
||||
* pull functionality ([923e7e7](https://github.com/AlexsJones/llmfit/commit/923e7e7463dd2bd53b6438ad3c8f2eb1f7a45af4))
|
||||
* release plumbing ([7d21719](https://github.com/AlexsJones/llmfit/commit/7d217192bc1638f7ff69a22c2467d7d86da96641))
|
||||
* release plumbing ([3accbb4](https://github.com/AlexsJones/llmfit/commit/3accbb42c99321fb6f8ade9d2f07af0fee93ed9e))
|
||||
* reworked available models for download ([9adc84f](https://github.com/AlexsJones/llmfit/commit/9adc84f3041dca14fdcdc4437409b2b81eaca5a3))
|
||||
* support for windows vulkan ([cc0fd61](https://github.com/AlexsJones/llmfit/commit/cc0fd619fa31e01c398c3c23f45aa915005670c8))
|
||||
* supporting 94 models ([a652be3](https://github.com/AlexsJones/llmfit/commit/a652be31dd0cbe36f89572de7022e2a145fb3788))
|
||||
* updated build actions ([1e65fdd](https://github.com/AlexsJones/llmfit/commit/1e65fddecb5f183870ddf1aa865dcaddba47523a))
|
||||
* updated images ([9141109](https://github.com/AlexsJones/llmfit/commit/9141109f753ef38eb2b2eb5c604edb6ee0d7e371))
|
||||
* updated models ([2d6c1d6](https://github.com/AlexsJones/llmfit/commit/2d6c1d66708186c0a21cb2f082a5b4e2fb03db90))
|
||||
* updated tui to support multiple providers better and also multiple GPU support ([a3ca0bd](https://github.com/AlexsJones/llmfit/commit/a3ca0bd64647fa958c15bb7038a9e02df175fe67))
|
||||
* updated urls ([f75ec27](https://github.com/AlexsJones/llmfit/commit/f75ec2750f325ff73725e5b8b194ba854c8579e9))
|
||||
* updated version ([2cfc73e](https://github.com/AlexsJones/llmfit/commit/2cfc73ebdb6214f801e32880ff6451b2809bbb45))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* correctly estimate VRAM for APU integrated GPUs ([72c8cb0](https://github.com/AlexsJones/llmfit/commit/72c8cb0e7873e0a8bcf4a10aee877bc38555299c))
|
||||
* correctly estimate VRAM for APU integrated GPUs (Radeon Graphics) ([8da5c2a](https://github.com/AlexsJones/llmfit/commit/8da5c2a0443b73a3ac78ac087b0f08acdba6aaa9)), closes [#25](https://github.com/AlexsJones/llmfit/issues/25)
|
||||
* update OpenClaw skill to match actual CLI output ([f38a0e5](https://github.com/AlexsJones/llmfit/commit/f38a0e56ef332bde8f3b03f8b06b5982fe90c1cc))
|
||||
* update OpenClaw skill to match actual CLI output ([e1adbfd](https://github.com/AlexsJones/llmfit/commit/e1adbfd0abd786bc7a99496f20a7f81070bc8fe3))
|
||||
|
||||
## [0.3.6](https://github.com/AlexsJones/llmfit/compare/llmfit-v0.3.5...llmfit-v0.3.6) (2026-02-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* release plumbing ([7d21719](https://github.com/AlexsJones/llmfit/commit/7d217192bc1638f7ff69a22c2467d7d86da96641))
|
||||
* release plumbing ([3accbb4](https://github.com/AlexsJones/llmfit/commit/3accbb42c99321fb6f8ade9d2f07af0fee93ed9e))
|
||||
|
||||
## [0.3.5](https://github.com/AlexsJones/llmfit/compare/llmfit-v0.3.4...llmfit-v0.3.5) (2026-02-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --memory flag to override GPU VRAM autodetection ([9a02f6e](https://github.com/AlexsJones/llmfit/commit/9a02f6e1616f59783ccff5b007c25213854f63b9))
|
||||
* add --memory flag to override GPU VRAM autodetection ([39c5486](https://github.com/AlexsJones/llmfit/commit/39c5486aa3d94f9b9ef36e29642b64d848d0d2b0))
|
||||
* add 15 popular models from HuggingFace ([128a020](https://github.com/AlexsJones/llmfit/commit/128a020323897a67ed5d12dd397bcf4924a6bf51))
|
||||
* Add 15 popular models from HuggingFace (33→48 models) ([c45606b](https://github.com/AlexsJones/llmfit/commit/c45606bdb235b6bfe616bb616b1364a97e76f0c1))
|
||||
* add homebrew tap support and update release workflow ([db09473](https://github.com/AlexsJones/llmfit/commit/db094734288d17a49d9c3c5c99859fe0d7dc976d))
|
||||
* added arc support ([b5892fc](https://github.com/AlexsJones/llmfit/commit/b5892fc2ff313e71f57b7d793c7444d2aaadc0bd))
|
||||
* added logo ([c21d416](https://github.com/AlexsJones/llmfit/commit/c21d4168f2bcd6da878848f9a6f97179d558606b))
|
||||
* added moe ([ac7ffe4](https://github.com/AlexsJones/llmfit/commit/ac7ffe4ed79eb22ec43cf7bc20e8cd8d102d16a9))
|
||||
* adding release please ([f2bfc7f](https://github.com/AlexsJones/llmfit/commit/f2bfc7fcf2587b74e05d8ad9d1041be6de456e69))
|
||||
* append (WSL) to RAM label in tui when running under WSL ([e0397cf](https://github.com/AlexsJones/llmfit/commit/e0397cf51025b393b0d4024c4ae67200ee206390))
|
||||
* caught some unavailable models on ollama ([b9f38da](https://github.com/AlexsJones/llmfit/commit/b9f38da9579040a7c2bada55838c5541474883ca))
|
||||
* caught some unavailable models on ollama ([c0f7c20](https://github.com/AlexsJones/llmfit/commit/c0f7c20f61cdd9ae692de6ca66344befba2fafa9))
|
||||
* detect installed Ollama models and support pulling from TUI ([4159aaf](https://github.com/AlexsJones/llmfit/commit/4159aaf304b3b421679f8231cf574465783d5b41))
|
||||
* first pass ([855ad3d](https://github.com/AlexsJones/llmfit/commit/855ad3d34160cce6200c0ff128c34bcdcb0b922b))
|
||||
* fixed up skill ([fcb712a](https://github.com/AlexsJones/llmfit/commit/fcb712a98ac785ad83ad689d5300f17cb80a3f1c))
|
||||
* fixed up skill ([1f7d1de](https://github.com/AlexsJones/llmfit/commit/1f7d1de547a31202b9d34dd62bf543f5a22b2de7))
|
||||
* fixing vram on apple bug ([5e08754](https://github.com/AlexsJones/llmfit/commit/5e087549c7c1523f4d5df72bd8a915330498a795))
|
||||
* fixing vram on apple bug ([b3deca1](https://github.com/AlexsJones/llmfit/commit/b3deca1d9eac16283d0e9269c68a1af1dfc871ab))
|
||||
* fixing vram on apple bug ([92ddb0e](https://github.com/AlexsJones/llmfit/commit/92ddb0e82579c6018d1acb4e3dfbe1df7d582605))
|
||||
* fixing vram on apple bug ([42b2081](https://github.com/AlexsJones/llmfit/commit/42b2081577bed23176c0f87d1ad0b142cce23872))
|
||||
* improvements based on [#12](https://github.com/AlexsJones/llmfit/issues/12) ([5428ef8](https://github.com/AlexsJones/llmfit/commit/5428ef8cdd42e88bced1459b55b480aab767637c))
|
||||
* increased model count ([156b29d](https://github.com/AlexsJones/llmfit/commit/156b29deb077a1d66948254b370597a118fd5daf))
|
||||
* increment version ([283bebb](https://github.com/AlexsJones/llmfit/commit/283bebb8eca5da2fc7124b665ae773fda48aed93))
|
||||
* overall to the scoring system ([f475938](https://github.com/AlexsJones/llmfit/commit/f4759381d23b834e0a42a4699d23fb3f858fe677))
|
||||
* overall to the scoring system ([b0696cf](https://github.com/AlexsJones/llmfit/commit/b0696cf297f1cb11247493355406d8b9c56510db))
|
||||
* overall to the scoring system ([37e2e10](https://github.com/AlexsJones/llmfit/commit/37e2e10076f450f79165d92541baf04957ec2fe9))
|
||||
* pull functionality ([923e7e7](https://github.com/AlexsJones/llmfit/commit/923e7e7463dd2bd53b6438ad3c8f2eb1f7a45af4))
|
||||
* reworked available models for download ([9adc84f](https://github.com/AlexsJones/llmfit/commit/9adc84f3041dca14fdcdc4437409b2b81eaca5a3))
|
||||
* support for windows vulkan ([cc0fd61](https://github.com/AlexsJones/llmfit/commit/cc0fd619fa31e01c398c3c23f45aa915005670c8))
|
||||
* supporting 94 models ([a652be3](https://github.com/AlexsJones/llmfit/commit/a652be31dd0cbe36f89572de7022e2a145fb3788))
|
||||
* updated build actions ([1e65fdd](https://github.com/AlexsJones/llmfit/commit/1e65fddecb5f183870ddf1aa865dcaddba47523a))
|
||||
* updated images ([9141109](https://github.com/AlexsJones/llmfit/commit/9141109f753ef38eb2b2eb5c604edb6ee0d7e371))
|
||||
* updated models ([2d6c1d6](https://github.com/AlexsJones/llmfit/commit/2d6c1d66708186c0a21cb2f082a5b4e2fb03db90))
|
||||
* updated tui to support multiple providers better and also multiple GPU support ([a3ca0bd](https://github.com/AlexsJones/llmfit/commit/a3ca0bd64647fa958c15bb7038a9e02df175fe67))
|
||||
* updated urls ([f75ec27](https://github.com/AlexsJones/llmfit/commit/f75ec2750f325ff73725e5b8b194ba854c8579e9))
|
||||
* updated version ([2cfc73e](https://github.com/AlexsJones/llmfit/commit/2cfc73ebdb6214f801e32880ff6451b2809bbb45))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* correctly estimate VRAM for APU integrated GPUs ([72c8cb0](https://github.com/AlexsJones/llmfit/commit/72c8cb0e7873e0a8bcf4a10aee877bc38555299c))
|
||||
* correctly estimate VRAM for APU integrated GPUs (Radeon Graphics) ([8da5c2a](https://github.com/AlexsJones/llmfit/commit/8da5c2a0443b73a3ac78ac087b0f08acdba6aaa9)), closes [#25](https://github.com/AlexsJones/llmfit/issues/25)
|
||||
* update OpenClaw skill to match actual CLI output ([f38a0e5](https://github.com/AlexsJones/llmfit/commit/f38a0e56ef332bde8f3b03f8b06b5982fe90c1cc))
|
||||
* update OpenClaw skill to match actual CLI output ([e1adbfd](https://github.com/AlexsJones/llmfit/commit/e1adbfd0abd786bc7a99496f20a7f81070bc8fe3))
|
||||
@@ -0,0 +1,17 @@
|
||||
# Code of Conduct
|
||||
|
||||
This project adopts the **Contributor Covenant v2.1** as its code of conduct.
|
||||
|
||||
The full text is available at: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
|
||||
|
||||
## Summary
|
||||
|
||||
We are committed to making participation in this project a welcoming experience for everyone regardless of background or identity. We expect all contributors to act professionally and respectfully.
|
||||
|
||||
## Reporting
|
||||
|
||||
If you experience or witness behavior that violates this code of conduct, please report it by opening an issue or contacting the maintainers directly.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Project maintainers are responsible for clarifying standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior they deem inappropriate.
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
# Contributing to llmfit
|
||||
|
||||
Thanks for your interest in contributing! Whether it's a bug fix, new feature, model addition, or documentation improvement, we appreciate the help.
|
||||
|
||||
## Getting started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- **Rust** (stable toolchain, edition 2024, MSRV 1.85+)
|
||||
- **Python 3** (for model database scripts — stdlib only, no pip dependencies)
|
||||
- **Git**
|
||||
|
||||
### Building from source
|
||||
|
||||
```sh
|
||||
git clone https://github.com/AlexsJones/llmfit.git
|
||||
cd llmfit
|
||||
make build # debug build
|
||||
make release # release build
|
||||
```
|
||||
|
||||
### Running
|
||||
|
||||
```sh
|
||||
make run # TUI mode (default)
|
||||
cargo run -- --cli # classic table output
|
||||
cargo run -- system # show detected hardware
|
||||
cargo run -- fit --perfect # show best-fit models
|
||||
cargo run -- search "llama" # search models
|
||||
```
|
||||
|
||||
### Useful commands
|
||||
|
||||
```sh
|
||||
make test # run all tests
|
||||
make fmt # format code (cargo fmt)
|
||||
make clippy # run linter (cargo clippy)
|
||||
make check # fast compilation check
|
||||
```
|
||||
|
||||
## Project structure
|
||||
|
||||
llmfit is a Rust workspace with three crates:
|
||||
|
||||
| Crate | Description |
|
||||
|-------|-------------|
|
||||
| `llmfit-core` | Core library — hardware detection, model database, fit analysis |
|
||||
| `llmfit-tui` | Terminal user interface (ratatui + crossterm) |
|
||||
| `llmfit-desktop` | Desktop app (Tauri) |
|
||||
|
||||
Supporting directories:
|
||||
|
||||
- `scripts/` — Python utilities for scraping HuggingFace and Docker model metadata
|
||||
- `data/` — Generated JSON model databases (do not edit manually)
|
||||
- `llmfit-python/` — Python bindings
|
||||
- `llmfit-web/` — Web interface
|
||||
|
||||
For a deeper dive into the architecture, see [AGENTS.md](AGENTS.md).
|
||||
|
||||
## How to contribute
|
||||
|
||||
### Reporting bugs
|
||||
|
||||
Open an [issue](https://github.com/AlexsJones/llmfit/issues) with:
|
||||
|
||||
- What you expected to happen
|
||||
- What actually happened
|
||||
- Your OS, hardware (GPU model, RAM), and llmfit version (`llmfit --version`)
|
||||
- Steps to reproduce
|
||||
|
||||
### Suggesting features
|
||||
|
||||
Start a [discussion](https://github.com/AlexsJones/llmfit/discussions) or open an issue. We're happy to chat about ideas before you invest time coding.
|
||||
|
||||
### Submitting a pull request
|
||||
|
||||
1. **Fork** the repo and create a branch from `main`.
|
||||
2. Make your changes.
|
||||
3. Run `cargo fmt` — most CI failures are from unformatted code.
|
||||
4. Run `make clippy` and fix any warnings.
|
||||
5. Run `make test` to verify nothing is broken.
|
||||
6. Open a PR against `main` with a clear description of what and why.
|
||||
|
||||
Keep PRs focused. One bug fix or feature per PR is easier to review than a combined change.
|
||||
|
||||
### Adding a new model
|
||||
|
||||
1. Add the model's HuggingFace repo ID (e.g., `meta-llama/Llama-3.1-8B`) to the `TARGET_MODELS` list in `scripts/scrape_hf_models.py`.
|
||||
2. If the model is gated (requires HF authentication), add a fallback entry to the `FALLBACKS` dict in the same script.
|
||||
3. Run `make update-models` to regenerate the database and rebuild.
|
||||
4. Verify with `./target/release/llmfit list`.
|
||||
5. Update [MODELS.md](MODELS.md) if needed.
|
||||
6. Update [llmfit-core/data/schema.json](llmfit-core/data/schema.json) if the model has new or unique metadata fields.
|
||||
7. Open a PR.
|
||||
|
||||
## Code guidelines
|
||||
|
||||
- **No `unsafe` code.**
|
||||
- No `.unwrap()` on user-facing paths. Use proper error handling or `expect()` with a descriptive message for internal invariants only.
|
||||
- Keep TUI rendering stateless — `tui_ui::draw()` must not mutate application state.
|
||||
- Prefer well-maintained crates with minimal transitive dependencies.
|
||||
- The Python scraper uses only stdlib (`urllib`, `json`). Do not add pip dependencies.
|
||||
|
||||
## Code of conduct
|
||||
|
||||
This project follows the [Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior by opening an issue.
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the [MIT License](LICENSE).
|
||||
Generated
+7103
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
[workspace]
|
||||
members = ["llmfit-core", "llmfit-tui", "llmfit-desktop"]
|
||||
default-members = ["llmfit-core", "llmfit-tui"]
|
||||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.1.2"
|
||||
license = "MIT"
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
# Multi-stage build for llmfit
|
||||
# Stage 1: Build the Rust binary
|
||||
# rustc >= 1.95 required: sysinfo 0.39.x bumped its MSRV to 1.95.
|
||||
# Pin the Debian release to match the runtime stage (bookworm). The default
|
||||
# rust:1.95-slim base tracks trixie (glibc 2.39), which links the binary
|
||||
# against symbols the bookworm runtime (glibc 2.36) does not provide, so the
|
||||
# binary fails to start with "GLIBC_2.39 not found". Keep both stages on the
|
||||
# same release so the linked glibc is always available at runtime.
|
||||
FROM rust:1.95-slim-bookworm AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /build
|
||||
|
||||
# Copy workspace configuration
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
|
||||
# Copy all workspace members
|
||||
COPY llmfit-core/ ./llmfit-core/
|
||||
COPY llmfit-tui/ ./llmfit-tui/
|
||||
COPY llmfit-desktop/ ./llmfit-desktop/
|
||||
|
||||
# Build release binary for llmfit-tui
|
||||
RUN cargo build --release -p llmfit
|
||||
|
||||
# Stage 2: Runtime image
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
# Install runtime dependencies for hardware detection
|
||||
RUN apt-get update && apt-get install -y \
|
||||
pciutils \
|
||||
lshw \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy the binary from builder
|
||||
COPY --from=builder /build/target/release/llmfit /usr/local/bin/llmfit
|
||||
|
||||
# Create a non-root user
|
||||
RUN useradd -m -u 1000 llmfit && \
|
||||
chown -R llmfit:llmfit /usr/local/bin/llmfit
|
||||
|
||||
USER llmfit
|
||||
|
||||
# Set default command to output JSON recommendations
|
||||
# In Kubernetes, this will run once per node and log results
|
||||
ENTRYPOINT ["/usr/local/bin/llmfit"]
|
||||
CMD ["recommend", "--json"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Alex Jones
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,259 @@
|
||||
# Supported Models
|
||||
|
||||
llmfit ships with a curated database of 106 LLM models from HuggingFace. All memory estimates assume Q4_K_M quantization (0.5 bytes per parameter) unless noted otherwise.
|
||||
|
||||
### 01.ai
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [01-ai/Yi-6B-Chat](https://huggingface.co/01-ai/Yi-6B-Chat) | 6.1B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [01-ai/Yi-34B-Chat](https://huggingface.co/01-ai/Yi-34B-Chat) | 34.4B | Q4_K_M | 4k | Instruction following, chat |
|
||||
|
||||
### Alibaba
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) | 600M | Q4_K_M | 40k | Lightweight, edge deployment |
|
||||
| [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B) | 873M | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3.5-0.8B-Base](https://huggingface.co/Qwen/Qwen3.5-0.8B-Base) | 873M | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen2.5-Coder-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct) | 1.5B | Q4_K_M | 32k | Code generation and completion |
|
||||
| [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) | 1.7B | Q4_K_M | 40k | Lightweight, edge deployment |
|
||||
| [Qwen/Qwen3.5-2B](https://huggingface.co/Qwen/Qwen3.5-2B) | 2.3B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base) | 2.3B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) | 3.8B | Q4_K_M | 32k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) | 4.0B | Q4_K_M | 40k | General purpose text generation |
|
||||
| [Qwen/Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B) | 4.7B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base) | 4.7B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) | 7.6B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | 7.6B | Q4_K_M | 32k | Code generation and completion |
|
||||
| [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) | 8.2B | Q4_K_M | 40k | General purpose text generation |
|
||||
| [Qwen/Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) | 8.3B | Q4_K_M | 32k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3.5-9B](https://huggingface.co/Qwen/Qwen3.5-9B) | 9.7B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3.5-9B-Base](https://huggingface.co/Qwen/Qwen3.5-9B-Base) | 9.7B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) | 14.8B | Q4_K_M | 128k | Instruction following, chat |
|
||||
| [Qwen/Qwen3-14B](https://huggingface.co/Qwen/Qwen3-14B) | 14.8B | Q4_K_M | 128k | General purpose text generation |
|
||||
| [Qwen/Qwen2.5-Coder-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct) | 14.8B | Q4_K_M | 32k | Code generation and completion |
|
||||
| [Qwen/Qwen3.5-27B](https://huggingface.co/Qwen/Qwen3.5-27B) | 27.8B | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | 30.5B (MoE) | Q4_K_M | 40k | Efficient MoE, general purpose |
|
||||
| [Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) | 36.0B (MoE) | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) | 32.5B | Q4_K_M | 128k | Instruction following, chat |
|
||||
| [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | 32.8B | Q4_K_M | 40k | General purpose text generation |
|
||||
| [Qwen/Qwen2.5-Coder-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct) | 32.8B | Q4_K_M | 32k | Code generation and completion |
|
||||
| [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) | 72.7B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [Qwen/Qwen3.5-122B-A10B](https://huggingface.co/Qwen/Qwen3.5-122B-A10B) | 125.1B (MoE) | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3-235B-A22B](https://huggingface.co/Qwen/Qwen3-235B-A22B) | 235B (MoE) | Q4_K_M | 40k | State-of-the-art, MoE architecture |
|
||||
| [Qwen/Qwen3.5-397B-A17B](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) | 403.4B (MoE) | Q4_K_M | 256k | Multimodal, vision and text |
|
||||
| [Qwen/Qwen3-Coder-480B-A35B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct) | 480B (MoE) | Q4_K_M | 256k | Code generation and completion |
|
||||
|
||||
### Allen Institute
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [allenai/OLMo-2-0325-32B-Instruct](https://huggingface.co/allenai/OLMo-2-0325-32B-Instruct) | 32B | Q4_K_M | 4k | Fully open-source, instruction following |
|
||||
|
||||
### Ant Group
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [inclusionAI/Ling-lite](https://huggingface.co/inclusionAI/Ling-lite) | 16.8B (MoE) | Q4_K_M | 128k | Efficient MoE, general purpose |
|
||||
|
||||
### BAAI
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [BAAI/bge-large-en-v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) | 335M | Q4_K_M | 512 | Text embeddings for RAG |
|
||||
|
||||
### Baidu
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [baidu/ERNIE-4.5-300B-A47B-Paddle](https://huggingface.co/baidu/ERNIE-4.5-300B-A47B-Paddle) | 300B (MoE) | Q4_K_M | 128k | Multilingual, reasoning |
|
||||
|
||||
### BigCode
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [bigcode/starcoder2-7b](https://huggingface.co/bigcode/starcoder2-7b) | 7.2B | Q4_K_M | 16k | Code generation and completion |
|
||||
| [bigcode/starcoder2-15b](https://huggingface.co/bigcode/starcoder2-15b) | 15.7B | Q4_K_M | 16k | Code generation and completion |
|
||||
|
||||
### BigScience
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [bigscience/bloom](https://huggingface.co/bigscience/bloom) | 176B | Q4_K_M | 2k | Multilingual text generation |
|
||||
|
||||
### Cohere
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) | 35B | Q4_K_M | 128k | RAG, tool use, agents |
|
||||
|
||||
### Community
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) | 1.1B | Q4_K_M | 2k | Instruction following, chat |
|
||||
|
||||
### DeepSeek
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [deepseek-ai/DeepSeek-R1-Distill-Qwen-7B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B) | 7.6B | Q4_K_M | 128k | Advanced reasoning, chain-of-thought |
|
||||
| [deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct) | 16B (MoE) | Q4_K_M | 128k | Code generation and completion |
|
||||
| [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) | 32.8B | Q4_K_M | 128k | Advanced reasoning, chain-of-thought |
|
||||
| [deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) | 671B (MoE) | Q4_K_M | 128k | Advanced reasoning, chain-of-thought |
|
||||
| [deepseek-ai/DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) | 685B (MoE) | Q4_K_M | 128k | State-of-the-art, MoE architecture |
|
||||
|
||||
### Google
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it) | 1B | Q4_K_M | 32k | Lightweight, edge deployment |
|
||||
| [google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it) | 2.6B | Q4_K_M | 4k | General purpose text generation |
|
||||
| [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it) | 4B | Q4_K_M | 128k | Lightweight, general purpose |
|
||||
| [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) | 9.2B | Q4_K_M | 4k | General purpose text generation |
|
||||
| [google/gemma-3-12b-it](https://huggingface.co/google/gemma-3-12b-it) | 12B | Q4_K_M | 128k | Multimodal, vision and text |
|
||||
| [google/gemma-3-27b-it](https://huggingface.co/google/gemma-3-27b-it) | 27B | Q4_K_M | 128k | General purpose text generation |
|
||||
| [google/gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) | 27.2B | Q4_K_M | 4k | General purpose text generation |
|
||||
|
||||
### HuggingFace
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) | 7.2B | Q4_K_M | 32k | General purpose text generation |
|
||||
|
||||
### IBM
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [ibm-granite/granite-4.0-h-micro](https://huggingface.co/ibm-granite/granite-4.0-h-micro) | 3B | Q4_K_M | 128k | Enterprise, hybrid Mamba/transformer |
|
||||
| [ibm-granite/granite-4.0-h-tiny](https://huggingface.co/ibm-granite/granite-4.0-h-tiny) | 7B (MoE) | Q4_K_M | 128k | Enterprise, hybrid Mamba/transformer |
|
||||
| [ibm-granite/granite-3.1-8b-instruct](https://huggingface.co/ibm-granite/granite-3.1-8b-instruct) | 8.1B | Q4_K_M | 128k | Enterprise, instruction following |
|
||||
| [ibm-granite/granite-4.0-h-small](https://huggingface.co/ibm-granite/granite-4.0-h-small) | 32B (MoE) | Q4_K_M | 128k | Enterprise, hybrid Mamba/transformer |
|
||||
|
||||
### LMSYS
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [lmsys/vicuna-7b-v1.5](https://huggingface.co/lmsys/vicuna-7b-v1.5) | 7.0B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [lmsys/vicuna-13b-v1.5](https://huggingface.co/lmsys/vicuna-13b-v1.5) | 13.0B | Q4_K_M | 4k | Instruction following, chat |
|
||||
|
||||
### Meituan
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [meituan/LongCat-Flash](https://huggingface.co/meituan/LongCat-Flash) | 560B (MoE) | Q4_K_M | 512k | Long context MoE |
|
||||
|
||||
### Meta
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [meta-llama/Llama-3.2-1B](https://huggingface.co/meta-llama/Llama-3.2-1B) | 1.2B | Q4_K_M | 4k | General purpose text generation |
|
||||
| [meta-llama/Llama-3.2-3B](https://huggingface.co/meta-llama/Llama-3.2-3B) | 3.2B | Q4_K_M | 4k | General purpose text generation |
|
||||
| [meta-llama/CodeLlama-7b-Instruct-hf](https://huggingface.co/meta-llama/CodeLlama-7b-Instruct-hf) | 6.7B | Q4_K_M | 4k | Code generation and completion |
|
||||
| [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) | 8.0B | Q4_K_M | 4k | General purpose text generation |
|
||||
| [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | 8.0B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) | 10.7B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [meta-llama/CodeLlama-13b-Instruct-hf](https://huggingface.co/meta-llama/CodeLlama-13b-Instruct-hf) | 13.0B | Q4_K_M | 4k | Code generation and completion |
|
||||
| [meta-llama/CodeLlama-34b-Instruct-hf](https://huggingface.co/meta-llama/CodeLlama-34b-Instruct-hf) | 33.7B | Q4_K_M | 4k | Code generation and completion |
|
||||
| [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | 70.6B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | 70.6B | Q4_K_M | 128k | Instruction following, chat |
|
||||
| [meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct) | 109B (MoE) | Q4_K_M | 10M | Multimodal, vision and text |
|
||||
| [meta-llama/Llama-4-Maverick-17B-128E-Instruct](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct) | 400B (MoE) | Q4_K_M | 1M | Multimodal, vision and text |
|
||||
| [meta-llama/Llama-3.1-405B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct) | 405.9B | Q4_K_M | 4k | Instruction following, chat |
|
||||
|
||||
### Microsoft
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [microsoft/phi-3-mini-4k-instruct](https://huggingface.co/microsoft/phi-3-mini-4k-instruct) | 3.8B | Q4_K_M | 4k | Lightweight, edge deployment |
|
||||
| [microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) | 3.8B | Q4_K_M | 128k | Lightweight, long context |
|
||||
| [microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct) | 3.8B | Q4_K_M | 128k | Lightweight, edge deployment |
|
||||
| [microsoft/Orca-2-7b](https://huggingface.co/microsoft/Orca-2-7b) | 7.0B | Q4_K_M | 4k | Reasoning, step-by-step solutions |
|
||||
| [microsoft/Orca-2-13b](https://huggingface.co/microsoft/Orca-2-13b) | 13.0B | Q4_K_M | 4k | Reasoning, step-by-step solutions |
|
||||
| [microsoft/phi-4](https://huggingface.co/microsoft/phi-4) | 14B | Q4_K_M | 16k | Reasoning, STEM, code generation |
|
||||
| [microsoft/Phi-3-medium-14b-instruct](https://huggingface.co/microsoft/Phi-3-medium-14b-instruct) | 14B | Q4_K_M | 4k | Balanced performance and size |
|
||||
|
||||
### Mistral AI
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 7.2B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410) | 8.0B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [mistralai/Mistral-Nemo-Instruct-2407](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407) | 12.2B | Q4_K_M | 128k | Instruction following, chat |
|
||||
| [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) | 24B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503) | 24B | Q4_K_M | 128k | Multimodal, vision and text |
|
||||
| [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) | 46.7B (MoE) | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [mistralai/Mistral-Large-Instruct-2407](https://huggingface.co/mistralai/Mistral-Large-Instruct-2407) | 123B | Q4_K_M | 128k | Large-scale instruction following |
|
||||
| [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1) | 140.6B (MoE) | Q4_K_M | 64k | Large MoE, instruction following |
|
||||
|
||||
### Moonshot
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [moonshotai/Kimi-K2-Instruct](https://huggingface.co/moonshotai/Kimi-K2-Instruct) | 1000B (MoE) | Q4_K_M | 128k | Large MoE, reasoning |
|
||||
|
||||
### Nomic
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | 137M | F16 | 8k | Text embeddings for RAG |
|
||||
|
||||
### NousResearch
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO](https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO) | 46.7B (MoE) | Q4_K_M | 32k | General purpose text generation |
|
||||
|
||||
### OpenChat
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [openchat/openchat-3.5-0106](https://huggingface.co/openchat/openchat-3.5-0106) | 7.0B | Q4_K_M | 8k | Instruction following, chat |
|
||||
|
||||
### Rednote
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [rednote-hilab/dots.llm1.inst](https://huggingface.co/rednote-hilab/dots.llm1.inst) | 142B (MoE) | Q4_K_M | 128k | MoE, general purpose |
|
||||
|
||||
### Stability AI
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [stabilityai/stablelm-2-1_6b-chat](https://huggingface.co/stabilityai/stablelm-2-1_6b-chat) | 1.6B | Q4_K_M | 4k | Instruction following, chat |
|
||||
|
||||
### TII
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) | 7.2B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [tiiuae/Falcon3-7B-Instruct](https://huggingface.co/tiiuae/Falcon3-7B-Instruct) | 7.5B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct) | 10.3B | Q4_K_M | 32k | Instruction following, chat |
|
||||
| [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) | 40.0B | Q4_K_M | 2k | Instruction following, chat |
|
||||
| [tiiuae/falcon-180B-chat](https://huggingface.co/tiiuae/falcon-180B-chat) | 180B | Q4_K_M | 2k | Large-scale instruction following |
|
||||
|
||||
### Upstage
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [upstage/SOLAR-10.7B-Instruct-v1.0](https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0) | 10.7B | Q4_K_M | 4k | High-performance instruction following |
|
||||
|
||||
### WizardLM
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [WizardLMTeam/WizardLM-13B-V1.2](https://huggingface.co/WizardLMTeam/WizardLM-13B-V1.2) | 13.0B | Q4_K_M | 4k | Instruction following, chat |
|
||||
| [WizardLMTeam/WizardCoder-15B-V1.0](https://huggingface.co/WizardLMTeam/WizardCoder-15B-V1.0) | 15.5B | Q4_K_M | 8k | Code generation and completion |
|
||||
|
||||
### xAI
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [xai-org/grok-1](https://huggingface.co/xai-org/grok-1) | 314B (MoE) | Q4_K_M | 8k | Large MoE, general purpose |
|
||||
|
||||
### Zhipu AI
|
||||
|
||||
| Model | Parameters | Quantization | Context | Use Case |
|
||||
|-------|-----------|--------------|---------|----------|
|
||||
| [THUDM/glm-4-9b-chat](https://huggingface.co/THUDM/glm-4-9b-chat) | 9B | Q4_K_M | 128k | Multilingual, instruction following |
|
||||
@@ -0,0 +1,76 @@
|
||||
# Makefile for llmfit
|
||||
# Convenience commands for building, testing, and updating the model database
|
||||
|
||||
.PHONY: help build release clean run test update-models update-docker-models update-catalogs check fmt clippy install
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "llmfit - LLM Model Fit Analyzer"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " make build - Build debug binary"
|
||||
@echo " make release - Build release binary"
|
||||
@echo " make run - Run in TUI mode (debug)"
|
||||
@echo " make test - Run all unit tests"
|
||||
@echo " make update-models - Fetch latest model data from HuggingFace"
|
||||
@echo " make update-docker-models - Refresh Docker Model Runner catalog"
|
||||
@echo " make update-catalogs - Refresh all catalogs (HF models + Docker) and rebuild"
|
||||
@echo " make check - Run cargo check"
|
||||
@echo " make fmt - Format code with rustfmt"
|
||||
@echo " make clippy - Run clippy linter"
|
||||
@echo " make clean - Remove build artifacts"
|
||||
@echo " make install - Install release binary to ~/.cargo/bin"
|
||||
@echo ""
|
||||
|
||||
# Build debug version
|
||||
build:
|
||||
cargo build
|
||||
|
||||
# Build release version
|
||||
release:
|
||||
cargo build --release
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
cargo clean
|
||||
|
||||
# Run in TUI mode
|
||||
run:
|
||||
cargo run
|
||||
|
||||
# Run tests
|
||||
test:
|
||||
cargo test
|
||||
|
||||
# Update model database from HuggingFace
|
||||
update-models:
|
||||
@./scripts/update_models.sh
|
||||
|
||||
# Refresh Docker Model Runner catalog from Docker Hub
|
||||
update-docker-models:
|
||||
python3 scripts/scrape_docker_models.py
|
||||
|
||||
# Refresh all catalogs (HF models + Docker) and rebuild
|
||||
# Runs HF scraper first (via update_models.sh which also rebuilds),
|
||||
# then Docker scraper (which depends on hf_models.json), then rebuilds again
|
||||
# to embed the updated Docker catalog.
|
||||
update-catalogs:
|
||||
@./scripts/update_models.sh
|
||||
python3 scripts/scrape_docker_models.py
|
||||
cargo build --release
|
||||
|
||||
# Check compilation without building
|
||||
check:
|
||||
cargo check
|
||||
|
||||
# Format code
|
||||
fmt:
|
||||
cargo fmt
|
||||
|
||||
# Run clippy
|
||||
clippy:
|
||||
cargo clippy -- -D warnings
|
||||
|
||||
# Install to ~/.cargo/bin
|
||||
install:
|
||||
cargo install --path .
|
||||
+992
@@ -0,0 +1,992 @@
|
||||
# llmfit
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/icon.svg" alt="llmfit アイコン" width="128" height="128">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="README.md">English</a> ·
|
||||
<a href="README.zh.md">中文</a> ·
|
||||
<b>日本語</b>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/AlexsJones/llmfit/actions/workflows/ci.yml"><img src="https://github.com/AlexsJones/llmfit/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
||||
<a href="https://crates.io/crates/llmfit"><img src="https://img.shields.io/crates/v/llmfit.svg" alt="Crates.io"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="ライセンス"></a>
|
||||
<a href="https://about.signpath.io"><img src="https://img.shields.io/badge/SignPath-signed-brightgreen?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxNiIgaGVpZ2h0PSIxNiIgZmlsbD0id2hpdGUiIHZpZXdCb3g9IjAgMCAxNiAxNiI+PHBhdGggZD0iTTEwLjA2NyA0LjU2N2wtNC43MzQgNC43MzMtMS40LTEuNGExIDEgMCAwIDAtMS40MTQgMS40MTRsMi4xIDIuMWExIDEgMCAwIDAgMS40MTQgMGw1LjQ0LTUuNDRhMSAxIDAgMCAwLTEuNDE0LTEuNDE0eiIvPjwvc3ZnPg==" alt="SignPath で署名済み"></a>
|
||||
</p>
|
||||
|
||||
> **📊 新機能：ベンチマーク＆共有 — あなたのマシンの実測値が、みんなの推定精度を高めます。** `llmfit bench --share` はあなたのハードウェアで実際の tok/s を計測し、PR としてプロジェクトに還元します — `gh` CLI もサードパーティのアカウントも不要です。実行結果はまずローカルに保存され（共有をスキップして、後からまとめてアップロードも可能）、自分の実測値はフィットテーブルの推定値を置き換えます。マージされた提出は次のリリースに同梱され、同一ハードウェアのユーザーはベンチマークを実行する前から実測 `✓` 値と校正済みの推定値を得られます。[共有をはじめる →](docs/cli.md#contributing-benchmarks-bench---share)
|
||||
>
|
||||
> *これまで：[llmfit 1.0 — すべての数値が検証可能になったリリース →](https://github.com/AlexsJones/llmfit/discussions/708)*
|
||||
|
||||
**数百のモデルとプロバイダー。自分のハードウェアで動くものを見つけるコマンドはひとつ。**
|
||||
|
||||
LLM モデルをシステムの RAM、CPU、GPU に合わせて最適化するターミナルツールです。ハードウェアを検出し、各モデルを品質・速度・適合度・コンテキストの各観点でスコアリングして、あなたのマシンで実際に快適に動くものを教えてくれます。
|
||||
|
||||
インタラクティブな TUI（デフォルト）と従来型の CLI モードを備えています。マルチ GPU 構成、MoE アーキテクチャ、動的な量子化選択、速度推定、ローカルランタイムプロバイダー（Ollama、llama.cpp、MLX、Docker Model Runner、LM Studio）に対応しています。
|
||||
|
||||
**新機能: [コミュニティリーダーボード](#コミュニティリーダーボード-b)（`b`）** — 同じハードウェアを使う他のユーザーから集まった実環境の tok/s、TTFT、VRAM 使用量を確認できます。[localmaxxing.com](https://localmaxxing.com) を利用し、推定パフォーマンスと実測パフォーマンスの差を埋めます。
|
||||
|
||||
その他: [ダウンロードマネージャー](#ダウンロードマネージャー-d)（`D`）、[詳細設定](#詳細設定-a)（`A`）、[ハードウェアシミュレーション](#ハードウェアシミュレーション-s) — `D` を押すとダウンロードの管理、履歴の閲覧、モデルの削除、ダウンロードディレクトリの設定ができます。`A` を押すと TPS 効率、実行モード係数、スコアリングの重みを調整できます。`S` を押すと別のハードウェアをシミュレーションできます。
|
||||
|
||||
> **姉妹プロジェクト:**
|
||||
> - [sympozium](https://github.com/sympozium-ai/sympozium/) — Kubernetes でエージェントを管理。
|
||||
> - [llmserve](https://github.com/AlexsJones/llmserve) — ローカル LLM モデルをサーブするためのシンプルな TUI。モデルを選び、バックエンドを選び、サーブする。
|
||||
> - [llama-panel](https://github.com/AlexsJones/llama-panel) — ローカルの llama-server インスタンスを管理するネイティブ macOS アプリ。
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## インストール
|
||||
|
||||
### Windows
|
||||
```sh
|
||||
scoop install llmfit
|
||||
```
|
||||
|
||||
Scoop がインストールされていない場合は、[Scoop インストールガイド](https://scoop.sh/)に従ってください。
|
||||
|
||||
### macOS / Linux
|
||||
|
||||
#### Homebrew
|
||||
|
||||
ビルド済みバイナリ（推奨。すべての macOS/Linux バージョンで動作）:
|
||||
```sh
|
||||
brew install AlexsJones/llmfit/llmfit
|
||||
```
|
||||
|
||||
または homebrew-core の formula から。bottle がない macOS バージョンではソースからビルドされます:
|
||||
```sh
|
||||
brew install llmfit
|
||||
```
|
||||
|
||||
#### MacPorts
|
||||
```sh
|
||||
port install llmfit
|
||||
```
|
||||
|
||||
#### クイックインストール
|
||||
```sh
|
||||
curl -fsSL https://llmfit.axjns.dev/install.sh | sh
|
||||
```
|
||||
|
||||
GitHub から最新リリースのバイナリをダウンロードし、`/usr/local/bin`（sudo がない場合は `~/.local/bin`）にインストールします。
|
||||
|
||||
**sudo なしで `~/.local/bin` にインストール:**
|
||||
```sh
|
||||
curl -fsSL https://llmfit.axjns.dev/install.sh | sh -s -- --local
|
||||
```
|
||||
|
||||
### uv / pip
|
||||
llmfit をインストールまたは更新するには:
|
||||
```sh
|
||||
uv tool install -U llmfit
|
||||
```
|
||||
|
||||
インストールせずに実行するには:
|
||||
```sh
|
||||
uvx llmfit
|
||||
```
|
||||
|
||||
pip や uv などのツールを使って、通常の方法で llmfit を Python パッケージとしてインストールすることもできます。
|
||||
|
||||
### Docker / Podman
|
||||
```sh
|
||||
docker run ghcr.io/alexsjones/llmfit
|
||||
```
|
||||
これは `llmfit recommend` コマンドの JSON を出力します。この JSON は `jq` でさらにクエリできます。
|
||||
```
|
||||
podman run ghcr.io/alexsjones/llmfit recommend --use-case coding | jq '.models[].name'
|
||||
```
|
||||
|
||||
### ソースから
|
||||
```sh
|
||||
git clone https://github.com/AlexsJones/llmfit.git
|
||||
cd llmfit
|
||||
cargo build --release
|
||||
# バイナリは target/release/llmfit にあります
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 使い方
|
||||
|
||||
### TUI（デフォルト）
|
||||
|
||||
```sh
|
||||
llmfit
|
||||
```
|
||||
|
||||
インタラクティブなターミナル UI を起動します。システムスペック（CPU、RAM、GPU 名、VRAM、バックエンド）が上部に表示されます。モデルは複合スコア順に並んだスクロール可能なテーブルに一覧表示されます。各行には、モデルのスコア、推定 tok/s、あなたのハードウェアに最適な量子化、実行モード、メモリ使用量、ユースケースカテゴリが表示されます。
|
||||
|
||||
| キー | アクション |
|
||||
|----------------------------|-----------------------------------------------------------------------|
|
||||
| `Up` / `Down` または `j` / `k` | モデルを移動 |
|
||||
| `/` | 検索モードに入る（名前、プロバイダー、パラメータ、ユースケースの部分一致） |
|
||||
| `Esc` または `Enter` | 検索モードを終了 |
|
||||
| `Ctrl-U` | 検索をクリア |
|
||||
| `f` | 適合フィルターを切り替え: All、Runnable、Perfect、Good、Marginal |
|
||||
| `a` | 利用可否フィルターを切り替え: All、GGUF Avail、Installed |
|
||||
| `s` | ソート列を切り替え: Score、Params、Mem%、Ctx、Date、Use Case |
|
||||
| `v` | Visual モードに入る（複数モデルを選択） |
|
||||
| `V` | Select モードに入る（列ベースのフィルタリング） |
|
||||
| `t` | カラーテーマを切り替え（自動保存） |
|
||||
| `p` | 選択したモデルの Plan モードを開く（ハードウェアプランニング） |
|
||||
| `P` | プロバイダーフィルターのポップアップを開く（入力で曖昧フィルタリング） |
|
||||
| `U` | ユースケースフィルターのポップアップを開く |
|
||||
| `C` | 機能フィルターのポップアップを開く |
|
||||
| `L` | ライセンスフィルターのポップアップを開く |
|
||||
| `R` | ランタイム/バックエンドフィルターのポップアップを開く（llama.cpp、MLX、vLLM） |
|
||||
| `S` | ハードウェアシミュレーションのポップアップを開く（RAM/VRAM/CPU を上書き） |
|
||||
| `A` | 詳細設定のポップアップを開く（効率や実行モード係数を調整） |
|
||||
| `b` | コミュニティリーダーボードビューを開く（localmaxxing.com） |
|
||||
| `I` | 推論ベンチビューを開く（あなたのモデルに対するローカル品質スコアリング） |
|
||||
| `h` | ヘルプポップアップを開く（すべてのキーバインド） |
|
||||
| `m` | 選択したモデルを比較対象にマーク |
|
||||
| `c` | 比較ビューを開く（マーク済み vs 選択中） |
|
||||
| `x` | 比較マークをクリア |
|
||||
| `i` | インストール済み優先ソートを切り替え（検出された任意のランタイムプロバイダー） |
|
||||
| `d` | 選択したモデルをダウンロード（複数利用可能ならプロバイダー選択） |
|
||||
| `D` | ダウンロードマネージャーを開く（履歴、削除、設定） |
|
||||
| `r` | ランタイムプロバイダーからインストール済みモデルを再読み込み |
|
||||
| `Enter` | 選択したモデルの詳細ビューを切り替え |
|
||||
| `PgUp` / `PgDn` | 10 件単位でスクロール |
|
||||
| `g` / `G` | 先頭 / 末尾にジャンプ |
|
||||
| `q` | 終了 |
|
||||
|
||||
### Vim 風モード
|
||||
|
||||
TUI は左下のステータスバーに表示される Vim 由来のモードを使用します。現在のモードによって、どのキーが有効かが決まります。
|
||||
|
||||
#### Normal モード
|
||||
|
||||
デフォルトのモードです。移動、検索、フィルタリング、ビューの表示を行います。上の表のすべてのキーがここで適用されます。
|
||||
|
||||
#### Visual モード（`v`）
|
||||
|
||||
一括比較のために連続したモデルの範囲を選択します。`v` を押して現在の行を起点に固定し、`j`/`k` または矢印キーで移動して選択範囲を広げます。選択された行はハイライトされます。
|
||||
|
||||
| キー | アクション |
|
||||
|---------------------|--------------------------------------------------------|
|
||||
| `j` / `k` または矢印 | 選択範囲を上下に拡張 |
|
||||
| `c` | 選択したすべてのモデルを比較（マルチ比較ビューを開く） |
|
||||
| `m` | 現在のモデルを 2 モデル比較用にマーク |
|
||||
| `Esc` または `v` | Visual モードを終了 |
|
||||
|
||||
マルチ比較ビューは、行が属性（Score、tok/s、Fit、Mem%、Params、Mode、Context、Quant など）、列がモデルとなるテーブルを表示します。最良の値はハイライトされます。画面に収まらないほど多くのモデルを選択した場合は、`h`/`l` または矢印キーで横スクロールできます。
|
||||
|
||||
#### Select モード（`V`）
|
||||
|
||||
列ベースのアクションです。`V`（shift-v）を押して Select モードに入り、`h`/`l` または矢印キーで列ヘッダー間を移動します。アクティブな列は視覚的にハイライトされます。`Enter` または `Space` を押すと、その列の現在のアクションが実行されます。
|
||||
|
||||
| 列 | フィルターアクション |
|
||||
|-------------------------------|---------------------------------------------------------------------------|
|
||||
| Inst | 利用可否フィルターを切り替え |
|
||||
| Model | 検索モードに入る |
|
||||
| Provider | プロバイダーポップアップを開く |
|
||||
| Params | パラメータサイズのバケットポップアップを開く（<3B、3-7B、7-14B、14-30B、30-70B、70B+） |
|
||||
| Score, tok/s, Mem%, Ctx, Date | その列でソート |
|
||||
| Quant | 量子化ポップアップを開く |
|
||||
| Mode | 実行モードポップアップを開く（GPU、MoE、CPU+GPU、CPU） |
|
||||
| Fit | 適合フィルターを切り替え |
|
||||
| Use Case | ユースケースポップアップを開く |
|
||||
|
||||
Select モードでも行のナビゲーションは引き続き機能するため、アクションを適用しながらその効果を確認できます: `j`/`k`、矢印キー、`Ctrl-U`、`Ctrl-D`、`PageUp`、`PageDown`、`Home`、`End`。`Esc` を押すと Normal モードに戻ります。
|
||||
|
||||
### TUI Plan モード（`p`）
|
||||
|
||||
Plan モードは通常の適合分析を反転させます。「何が自分のハードウェアに収まるか?」ではなく、「このモデル構成にはどんなハードウェアが必要か?」を推定します。
|
||||
|
||||
選択した行で `p` を押し、次のように操作します:
|
||||
|
||||
| キー | アクション |
|
||||
|------------------------|-----------------------------------------------------------|
|
||||
| `Tab` / `j` / `k` | 編集可能なフィールド間を移動（Context、Quant、Target TPS） |
|
||||
| `Left` / `Right` | 現在のフィールド内でカーソルを移動 |
|
||||
| 入力 | 現在のフィールドを編集 |
|
||||
| `Backspace` / `Delete` | 文字を削除 |
|
||||
| `Ctrl-U` | 現在のフィールドをクリア |
|
||||
| `Esc` または `q` | Plan モードを終了 |
|
||||
|
||||
Plan モードは以下の推定値を表示します:
|
||||
- 最小および推奨の VRAM/RAM/CPU コア数
|
||||
- 実行可能な実行パス（GPU、CPU オフロード、CPU のみ）
|
||||
- より良い適合目標に到達するためのアップグレード差分
|
||||
|
||||
### ハードウェアシミュレーション（`S`）
|
||||
|
||||
`S` を押すとハードウェアシミュレーションのポップアップが開きます。RAM、VRAM、CPU コア数を上書きして、別のターゲットハードウェアでどのモデルが収まるかを確認できます。すべてのモデルスコア、適合レベル、速度推定は、シミュレーションされたスペックに対して即座に再計算されます。
|
||||
|
||||
|
||||
|
||||
| キー | アクション |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `Tab` / `j` / `k` | RAM、VRAM、CPU フィールドを切り替え |
|
||||
| 数字を入力 | 選択したフィールドを編集 |
|
||||
| `Enter` | シミュレーションを適用 |
|
||||
| `Ctrl-R` | 実際に検出されたハードウェアにリセット |
|
||||
| `Esc` | キャンセルして閉じる |
|
||||
|
||||
シミュレーションが有効なときは、システムバーとステータスバーに `SIM` バッジが表示されます。リセットするまで、モデルテーブル全体がシミュレーションされたハードウェアを反映します。
|
||||
|
||||
### 詳細設定（`A`）
|
||||
|
||||
`A` を押すと詳細設定のポップアップが開きます。このパネルでは、TPS 推定、実行モードのペナルティ、複合スコアリングの背後にあるパラメータを調整できます。これは特定のモデル（例: Qwen3 30B）で tok/s が過大評価されていた[issue #449](https://github.com/AlexsJones/llmfit/issues/449)に対応するものです。
|
||||
|
||||
すべての変更は即座に適用され、モデルテーブルが再計算されます。`Esc` で確定して閉じるか、`Ctrl-R` でデフォルトにリセットします。
|
||||
|
||||
| フィールド | 説明 | デフォルト |
|
||||
|--------------------|-------------------------------------------------------------------------|---------|
|
||||
| **Efficiency** | 帯域幅ベースの TPS のグローバル効率係数。オーバーヘッドを考慮 | `0.55` |
|
||||
| **GPU factor** | 純粋な GPU 推論の速度倍率 | `1.0` |
|
||||
| **CPU Offload** | 重みがシステム RAM にあふれた場合の速度倍率 | `0.5` |
|
||||
| **MoE Offload** | Mixture-of-Experts のエキスパート切り替えの速度倍率 | `0.8` |
|
||||
| **Tensor Par** | テンソル並列推論の速度倍率 | `0.9` |
|
||||
| **CPU Only** | CPU のみの実行の速度倍率 | `0.3` |
|
||||
| **Context cap** | メモリ推定に使用する最大コンテキスト長（デフォルトの場合は空欄のまま） | `auto` |
|
||||
|
||||
| キー | アクション |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `Tab` / `j` / `k` | フィールドを切り替え |
|
||||
| 数字 / `.` を入力 | 選択したフィールドを編集 |
|
||||
| `Left` / `Right` | フィールド内でカーソルを移動 |
|
||||
| `Backspace` / `Delete` | 文字を削除 |
|
||||
| `Ctrl-U` | 現在のフィールドをクリア |
|
||||
| `Enter` | 変更を適用してすべてのスコアを再計算 |
|
||||
| `Esc` / `q` | 適用せずに閉じる |
|
||||
|
||||
### ダウンロードマネージャー（`D`）
|
||||
|
||||
`D` を押すとダウンロードマネージャービューが開きます。このフルスクリーンビューはメインのモデルテーブルに置き換わり、3 つのセクションを提供します:
|
||||
|
||||
- **Active Download** — 進行中の現在のダウンロードを、プログレスバー、モデル名、ステータスメッセージとともに表示します。
|
||||
- **Config** — GGUF モデルディレクトリを表示（および編集可能）します。設定したパスはセッションをまたいで永続化されます。
|
||||
- **History** — 過去のダウンロードのナビゲート可能なリスト（新しい順）を、モデル名、プロバイダー、ステータス、日付とともに表示します。失敗したダウンロードは履歴から削除でき、成功したダウンロードはプロバイダーから削除できます。
|
||||
|
||||
`Tab` / `Shift-Tab` でセクション間のフォーカスを切り替えます。
|
||||
|
||||
| キー | アクション |
|
||||
|------------------------|--------------------------------------------------|
|
||||
| `Tab` / `Shift-Tab` | フォーカスを切り替え: Active → Config → History |
|
||||
| `j` / `k` または矢印 | 履歴リストをナビゲート（History にフォーカス時） |
|
||||
| `x` | 選択したモデルを削除（確認を求められる） |
|
||||
| `y` / `n` | 削除を確認またはキャンセル |
|
||||
| `e` | ダウンロードディレクトリを編集（Config にフォーカス時） |
|
||||
| `Enter` | ディレクトリ編集を確定 |
|
||||
| `Esc` / `D` / `q` | 閉じてモデルテーブルに戻る |
|
||||
|
||||
失敗したダウンロード（例: 404 エラー）の場合、`x` は履歴からエントリを削除します。成功したダウンロードの場合は、プロバイダーからモデルを削除します（Ollama と llama.cpp でサポート）。
|
||||
|
||||
### コミュニティリーダーボード（`b`）
|
||||
|
||||
`b` を押すとコミュニティリーダーボードビューが開きます。llmfit の理論的な速度推定だけに頼るのではなく、このビューでは同じハードウェアを使う他のユーザーから集まった**実環境のパフォーマンスデータ** — 実測の tok/s、最初のトークンまでの時間、ピーク VRAM 使用量 — を表示します。
|
||||
|
||||
|
||||
|
||||
データはコミュニティのベンチマークデータベースである [localmaxxing.com](https://localmaxxing.com) から取得されます。ビューを開くと、llmfit はあなたのハードウェア（GPU モデル、VRAM ティア、Apple Silicon チップファミリー、OS）を自動検出し、一致する結果をクエリします。
|
||||
|
||||
| 列 | 説明 |
|
||||
|--------------|----------------------------------------------------------|
|
||||
| **Model** | HuggingFace モデル ID |
|
||||
| **Engine** | 使用された推論ランタイム（llama.cpp、vLLM、Ollama、MLX...） |
|
||||
| **Quant** | 量子化フォーマット（Q4_K_M、Q8_0 など） |
|
||||
| **tok/s** | 実測の出力トークン生成速度 |
|
||||
| **Total t/s**| 総スループット（プロンプト + 生成） |
|
||||
| **TTFT** | 最初のトークンまでの時間（レイテンシ） |
|
||||
| **VRAM** | 推論中のピークメモリ使用量 |
|
||||
| **Ctx** | ベンチマークで使用されたコンテキスト長 |
|
||||
| **User** | 投稿者（認証済みユーザーは `*` でマーク） |
|
||||
|
||||
| キー | アクション |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `j` / `k` または矢印 | 結果をナビゲート |
|
||||
| `H` | ハードウェアピッカーを開く（任意の GPU を閲覧） |
|
||||
| `r` | API から再取得 / リフレッシュ |
|
||||
| `b` / `q` / `Esc` | 閉じてモデルテーブルに戻る |
|
||||
|
||||
`H` を押すとハードウェアピッカーが開きます。これは 27 種類の人気 GPU とチップ（RTX 5090 から CPU のみまで、加えて Apple Silicon M1–M4 バリアント、AMD RX/MI シリーズ、NVIDIA データセンターカード）のスクロール可能なリストです。1 つ選ぶと、たとえそれが自分の使っているものでなくても、そのハードウェアのベンチマークを即座に読み込めます。「My Hardware (auto-detect)」を選ぶと自分のシステムに戻ります。
|
||||
|
||||
#### API キーの設定
|
||||
|
||||
公開ベンチマークは認証なしで利用できます。フルアクセスには [localmaxxing.com](https://localmaxxing.com) の API キーを指定します:
|
||||
|
||||
```sh
|
||||
# 環境変数経由（推奨）
|
||||
export LOCALMAXXING_API_KEY="bhk_your_key_here"
|
||||
llmfit
|
||||
|
||||
# または CLI フラグ経由
|
||||
llmfit --api-key "bhk_your_key_here"
|
||||
```
|
||||
|
||||
| 変数 | 説明 |
|
||||
|---|---|
|
||||
| `LOCALMAXXING_API_KEY` | localmaxxing.com API のベアラートークン |
|
||||
|
||||
### 推論ベンチ（`I`）
|
||||
|
||||
`I`（大文字）を押すと推論ベンチビューが開きます。これは**ローカルで実行中のプロバイダー** — Ollama、vLLM、MLX — に対して**ライブの推論ベンチマーク**を実行し、実際の推論リクエストで最初のトークンまでの時間（TTFT）、毎秒トークン数（TPS）、総レイテンシを測定します。
|
||||
|
||||
コミュニティリーダーボード（他のユーザーから集めたクラウドソースのデータを表示）とは異なり、推論ベンチはあなたの実際のハードウェアであなたの実際のモデルを測定します。
|
||||
|
||||
#### TUI での使い方
|
||||
|
||||
| キー | アクション |
|
||||
|-----|--------|
|
||||
| `I` | 推論ベンチを開く（プロバイダーを自動検出してベンチマークを実行） |
|
||||
| `I`（再度） | ベンチビュー内からベンチマークを再実行 |
|
||||
| `j` / `k` または矢印 | モデル結果をナビゲート |
|
||||
| `Enter` | 選択したモデルの詳細ビューを開く |
|
||||
| `r` | ルーティングマトリックスビューに切り替え |
|
||||
| `q` / `Esc` | ベンチビューを閉じる |
|
||||
|
||||
結果は `~/.config/llmfit/bench-cache.json` にキャッシュされ、次回以降は即座に読み込まれます。
|
||||
|
||||
#### CLI での使い方
|
||||
|
||||
```sh
|
||||
# プロバイダーを自動検出してベンチマーク
|
||||
llmfit bench
|
||||
|
||||
# 実行中のすべてのプロバイダーで検出されたすべてのモデルをベンチマーク
|
||||
llmfit bench --all
|
||||
|
||||
# Ollama 経由で特定のモデルをベンチマーク
|
||||
llmfit bench --provider ollama llama3.2
|
||||
|
||||
# エンドポイント URL を上書き
|
||||
llmfit bench --provider ollama --url http://my-server:11434 llama3.2
|
||||
|
||||
# vLLM エンドポイントを上書き
|
||||
llmfit bench --provider vllm --url http://localhost:8000
|
||||
|
||||
# JSON として出力（スクリプト用）
|
||||
llmfit bench --json
|
||||
|
||||
# 品質ベンチマークを実行（ルーティング用のロールベーススコアリング）
|
||||
llmfit bench --quality
|
||||
|
||||
# ルーティングマトリックスを出力
|
||||
llmfit bench --quality --routing
|
||||
```
|
||||
|
||||
#### 環境変数
|
||||
|
||||
| 変数 | デフォルト | 説明 |
|
||||
|---|---|---|
|
||||
| `OLLAMA_HOST` | `http://localhost:11434` | Ollama API のベース URL |
|
||||
| `VLLM_PORT` | `8000` | vLLM サーバーのポート（`http://localhost:$VLLM_PORT` として使用） |
|
||||
|
||||
### テーマ
|
||||
|
||||
`t` を押すと 10 種類の組み込みカラーテーマを切り替えられます。選択は `~/.config/llmfit/theme` に自動保存され、次回起動時に復元されます。
|
||||
|
||||
| テーマ | 説明 |
|
||||
|--------------------------|---------------------------------------------------|
|
||||
| **Default** | オリジナルの llmfit カラー |
|
||||
| **Dracula** | 暗い紫の背景にパステルのアクセント |
|
||||
| **Solarized** | Ethan Schoonover の Solarized Dark パレット |
|
||||
| **Nord** | 北極風の涼しげな青灰色のトーン |
|
||||
| **Monokai** | Monokai Pro の暖かいシンタックスカラー |
|
||||
| **Gruvbox** | 暖かいアースカラーのレトログルーヴパレット |
|
||||
| **Catppuccin Latte** | 🌻 ライトテーマ — 調和の取れたパステルの反転 |
|
||||
| **Catppuccin Frappé** | 🪴 低コントラストのダーク — 落ち着いた控えめな美学 |
|
||||
| **Catppuccin Macchiato** | 🌺 中コントラストのダーク — 穏やかで安らぐトーン |
|
||||
| **Catppuccin Mocha** | 🌿 最も暗いバリアント — 色彩豊かなアクセントで居心地よく |
|
||||
|
||||
### Web ダッシュボード
|
||||
|
||||
`llmfit` を非 JSON モードで実行すると、`0.0.0.0:8787` でバックグラウンドの Web ダッシュボードが自動的に起動します。同じネットワーク上の任意のブラウザで開けます:
|
||||
|
||||
```
|
||||
http://<your-machine-ip>:8787
|
||||
```
|
||||
|
||||
ホストやポートは環境変数で上書きできます:
|
||||
|
||||
```sh
|
||||
LLMFIT_DASHBOARD_HOST=0.0.0.0 LLMFIT_DASHBOARD_PORT=9000 llmfit
|
||||
```
|
||||
|
||||
| 変数 | デフォルト | 説明 |
|
||||
|---|---|---|
|
||||
| `LLMFIT_DASHBOARD_HOST` | `0.0.0.0` | ダッシュボードサーバーをバインドするインターフェース |
|
||||
| `LLMFIT_DASHBOARD_PORT` | `8787` | ダッシュボードサーバーをバインドするポート |
|
||||
|
||||
自動起動するダッシュボードを無効にするには、`--no-dashboard` を渡します:
|
||||
|
||||
```sh
|
||||
llmfit --no-dashboard
|
||||
```
|
||||
|
||||
### CLI モード
|
||||
|
||||
`--cli` または任意のサブコマンドを使うと、従来型のテーブル出力が得られます:
|
||||
|
||||
```sh
|
||||
# 適合度でランク付けされたすべてのモデルのテーブル
|
||||
llmfit --cli
|
||||
|
||||
# 完全に適合するモデルのみ、上位 5 件
|
||||
llmfit fit --perfect -n 5
|
||||
|
||||
# 検出されたシステムスペックを表示
|
||||
llmfit system
|
||||
|
||||
# データベース内のすべてのモデルを一覧表示
|
||||
llmfit list
|
||||
|
||||
# 名前、プロバイダー、サイズで検索
|
||||
llmfit search "llama 8b"
|
||||
|
||||
# 単一モデルの詳細ビュー
|
||||
llmfit info "Mistral-7B"
|
||||
|
||||
# 上位 5 件の推奨（JSON、エージェント/スクリプト消費用）
|
||||
llmfit recommend --json --limit 5
|
||||
|
||||
# ユースケースでフィルタリングした推奨
|
||||
llmfit recommend --json --use-case coding --limit 3
|
||||
|
||||
# 特定のランタイムを強制（Apple Silicon での自動 MLX 選択をバイパス）
|
||||
llmfit recommend --force-runtime llamacpp
|
||||
llmfit recommend --force-runtime llamacpp --use-case coding --limit 3
|
||||
|
||||
# 特定のモデル構成に必要なハードウェアをプラン
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192 --quant mlx-4bit
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192 --target-tps 25 --json
|
||||
|
||||
# ノードレベルの REST API として実行（クラスタースケジューラー / アグリゲーター用）
|
||||
llmfit serve --host 0.0.0.0 --port 8787
|
||||
```
|
||||
|
||||
### REST API（`llmfit serve`）
|
||||
|
||||
`llmfit serve` は、TUI/CLI で使われるのと同じ適合度/スコアリングデータを公開する HTTP API を起動します。これにはノードのフィルタリングや上位モデルの選択も含まれます。
|
||||
|
||||
```sh
|
||||
# 生存確認
|
||||
curl http://localhost:8787/health
|
||||
|
||||
# ノードのハードウェア情報
|
||||
curl http://localhost:8787/api/v1/system
|
||||
|
||||
# フィルター付きの完全な適合リスト
|
||||
curl "http://localhost:8787/api/v1/models?min_fit=marginal&runtime=llamacpp&sort=score&limit=20"
|
||||
|
||||
# 主要なスケジューリングエンドポイント: このノードで実行可能な上位モデル
|
||||
curl "http://localhost:8787/api/v1/models/top?limit=5&min_fit=good&use_case=coding"
|
||||
|
||||
# モデル名/プロバイダーのテキストで検索
|
||||
curl "http://localhost:8787/api/v1/models/Mistral?runtime=any"
|
||||
```
|
||||
|
||||
`models`/`models/top` でサポートされるクエリパラメータ:
|
||||
|
||||
- `limit`（または `n`）: 返される行の最大数
|
||||
- `perfect`: `true|false`（`true` で完全適合のみを強制）
|
||||
- `min_fit`: `perfect|good|marginal|too_tight`
|
||||
- `runtime`: `any|mlx|llamacpp`
|
||||
- `use_case`: `general|coding|reasoning|chat|multimodal|embedding`
|
||||
- `provider`: プロバイダーのテキストフィルター（部分文字列）
|
||||
- `search`: 名前/プロバイダー/サイズ/ユースケースにわたる自由テキストフィルター
|
||||
- `sort`: `score|tps|params|mem|ctx|date|use_case`
|
||||
- `include_too_tight`: 実行不可能な行を含める（`/top` ではデフォルト `false`、`/models` では `true`）
|
||||
- `max_context`: メモリ推定のためのリクエストごとのコンテキスト上限
|
||||
- `force_runtime`: `mlx|llamacpp|vllm` — 分析中の自動ランタイム選択を上書き
|
||||
|
||||
API の動作をローカルで検証:
|
||||
|
||||
```sh
|
||||
# サーバーを自動的に起動し、エンドポイント/スキーマ/フィルターのアサーションを実行
|
||||
python3 scripts/test_api.py --spawn
|
||||
|
||||
# またはすでに実行中のサーバーをテスト
|
||||
python3 scripts/test_api.py --base-url http://127.0.0.1:8787
|
||||
```
|
||||
|
||||
### ハードウェアの上書き
|
||||
|
||||
ハードウェアの自動検出は一部のシステム（例: 壊れた `nvidia-smi`、VM、パススルー構成）で失敗することがあります。また、別のターゲットハードウェアに対してモデルの適合度を評価したい場合もあるでしょう。`--memory`、`--ram`、`--cpu-cores` を使って検出された値を上書きできます:
|
||||
|
||||
```sh
|
||||
# GPU VRAM を上書き
|
||||
llmfit --memory=32G
|
||||
|
||||
# システム RAM を上書き
|
||||
llmfit --ram=128G
|
||||
|
||||
# CPU コア数を上書き
|
||||
llmfit --cpu-cores=16
|
||||
|
||||
# 上書きを組み合わせてターゲットハードウェアをシミュレーション
|
||||
llmfit --memory=24G --ram=64G --cpu-cores=8 fit
|
||||
llmfit --memory=24G --ram=64G system --json
|
||||
|
||||
# すべてのモードで動作: TUI、CLI、サブコマンド
|
||||
llmfit --memory=24G --cli
|
||||
llmfit --memory=24G fit --perfect -n 5
|
||||
llmfit --ram=64G recommend --json
|
||||
```
|
||||
|
||||
`--memory` と `--ram` で使える接尾辞: `G`/`GB`/`GiB`（ギガバイト）、`M`/`MB`/`MiB`（メガバイト）、`T`/`TB`/`TiB`（テラバイト）。大文字小文字は区別されません。GPU が検出されなかった場合、`--memory` は合成 GPU エントリを作成し、モデルが GPU 推論用にスコアリングされるようにします。統合メモリシステム（Apple Silicon）では、`--ram` は VRAM も更新します。VRAM を独立して上書きするには `--memory` を使ってください。
|
||||
|
||||
### 推定用のコンテキスト長の上限
|
||||
|
||||
`--max-context` を使うと、メモリ推定に使用するコンテキスト長を上限で制限できます（各モデルが公称する最大コンテキストは変更しません）:
|
||||
|
||||
```sh
|
||||
# 4K コンテキストでメモリ適合度を推定
|
||||
llmfit --max-context 4096 --cli
|
||||
|
||||
# サブコマンドで動作
|
||||
llmfit --max-context 8192 fit --perfect -n 5
|
||||
llmfit --max-context 16384 recommend --json --limit 5
|
||||
```
|
||||
|
||||
`--max-context` が設定されていない場合、llmfit は利用可能であれば `OLLAMA_CONTEXT_LENGTH` を使用します。
|
||||
|
||||
### JSON 出力
|
||||
|
||||
任意のサブコマンドに `--json` を追加すると、機械可読な出力が得られます:
|
||||
|
||||
```sh
|
||||
llmfit --json system # ハードウェアスペックを JSON で
|
||||
llmfit --json fit -n 10 # 上位 10 件の適合を JSON で
|
||||
llmfit recommend --json # 上位 5 件の推奨（recommend では JSON がデフォルト）
|
||||
llmfit plan "Qwen/Qwen2.5-Coder-0.5B-Instruct" --context 8192 --json
|
||||
```
|
||||
|
||||
`plan` の JSON には以下の安定したフィールドが含まれます:
|
||||
- リクエスト（`context`、`quantization`、`target_tps`）
|
||||
- 推定された最小/推奨ハードウェア
|
||||
- パスごとの実行可能性（`gpu`、`cpu_offload`、`cpu_only`）
|
||||
- アップグレード差分
|
||||
|
||||
---
|
||||
|
||||
## 仕組み
|
||||
|
||||
1. **ハードウェア検出** -- `sysinfo` で合計/利用可能 RAM を読み取り、CPU コアを数え、GPU を探索します:
|
||||
- **NVIDIA** -- `nvidia-smi` によるマルチ GPU サポート。検出されたすべての GPU の VRAM を集約します。レポートが失敗した場合は GPU モデル名から VRAM を推定してフォールバックします。
|
||||
- **AMD** -- `rocm-smi` で検出。
|
||||
- **Intel Arc** -- ディスクリート VRAM は sysfs 経由、統合は `lspci` 経由。
|
||||
- **Apple Silicon** -- `system_profiler` 経由の統合メモリ。VRAM = システム RAM。
|
||||
- **Ascend** -- `npu-smi` で検出。
|
||||
- **バックエンド検出** -- 速度推定のために、アクセラレーションバックエンド（CUDA、Metal、ROCm、SYCL、CPU ARM、CPU x86、Ascend）を自動的に識別します。
|
||||
|
||||
2. **モデルデータベース** -- HuggingFace API から取得した数百のモデルを `data/hf_models.json` に保存し、コンパイル時に埋め込みます。メモリ要件は、量子化階層（Q8_0 から Q2_K まで）にわたるパラメータ数から計算されます。VRAM は GPU 推論の主要な制約であり、システム RAM は CPU のみの実行のフォールバックです。
|
||||
|
||||
**MoE サポート** -- Mixture-of-Experts アーキテクチャを持つモデル（Mixtral、DeepSeek-V2/V3）は自動的に検出されます。トークンごとにアクティブになるのはエキスパートの一部のみなので、実効 VRAM 要件は総パラメータ数が示すよりもはるかに低くなります。例えば、Mixtral 8x7B は総パラメータ 46.7B ですが、トークンごとにアクティブになるのは約 12.9B のみで、エキスパートオフロードにより VRAM が 23.9 GB から約 6.6 GB に削減されます。
|
||||
|
||||
3. **動的量子化** -- 固定の量子化を仮定する代わりに、llmfit はあなたのハードウェアに収まる最高品質の量子化を試します。Q8_0（最高品質）から Q2_K（最も圧縮）までの階層をたどり、利用可能なメモリに収まる最高品質のものを選びます。フルコンテキストで何も収まらない場合は、半分のコンテキストで再試行します。
|
||||
|
||||
4. **多次元スコアリング** -- 各モデルは 4 つの次元（それぞれ 0〜100）でスコアリングされます:
|
||||
|
||||
| 次元 | 測定するもの |
|
||||
|-------------|--------------------------------------------------------------------------------|
|
||||
| **Quality** | パラメータ数、モデルファミリーの評判、量子化ペナルティ、タスク整合性 |
|
||||
| **Speed** | バックエンド、パラメータ、量子化に基づく推定トークン/秒 |
|
||||
| **Fit** | メモリ使用効率（最適点: 利用可能メモリの 50〜80%） |
|
||||
| **Context** | ユースケースに対するコンテキストウィンドウ能力 vs ターゲット |
|
||||
|
||||
各次元は重み付き複合スコアに統合されます。重みはユースケースカテゴリ（General、Coding、Reasoning、Chat、Multimodal、Embedding）によって異なります。例えば、Chat は Speed を高く重み付け（0.35）し、Reasoning は Quality を高く重み付け（0.55）します。モデルは複合スコアでランク付けされ、実行不可能なモデル（Too Tight）は常に最下位になります。
|
||||
|
||||
5. **速度推定** -- LLM 推論におけるトークン生成はメモリ帯域幅に律速されます。各トークンは VRAM からモデルの全重みを一度読み取る必要があります。GPU モデルが認識されると、llmfit はその実際のメモリ帯域幅を使ってスループットを推定します:
|
||||
|
||||
計算式: `(bandwidth_GB_s / model_size_GB) × efficiency_factor`
|
||||
|
||||
効率係数（0.55）とモードごとの速度倍率は、詳細設定ポップアップ（TUI の `A`）で調整できます。デフォルト値は、カーネルオーバーヘッド、KV キャッシュの読み取り、メモリコントローラーの影響を考慮しています。このアプローチは、llama.cpp の公開ベンチマーク（[Apple Silicon](https://github.com/ggml-org/llama.cpp/discussions/4167)、[NVIDIA T4](https://github.com/ggml-org/llama.cpp/discussions/4225)）および実環境の測定値に対して検証されています。
|
||||
|
||||
帯域幅ルックアップテーブルは、NVIDIA（コンシューマー + データセンター）、AMD（RDNA + CDNA）、Apple Silicon ファミリーにわたる約 80 の GPU をカバーしています。
|
||||
|
||||
認識されない GPU の場合、llmfit はバックエンドごとの速度定数にフォールバックします:
|
||||
|
||||
| バックエンド | 速度定数 |
|
||||
|--------------|----------------|
|
||||
| CUDA | 220 |
|
||||
| Metal | 160 |
|
||||
| ROCm | 180 |
|
||||
| SYCL | 100 |
|
||||
| CPU (ARM) | 90 |
|
||||
| CPU (x86) | 70 |
|
||||
| NPU (Ascend) | 390 |
|
||||
|
||||
フォールバック計算式: `K / params_b × quant_speed_multiplier`。モードごとのペナルティは詳細設定ポップアップ（TUI の `A`）で調整できます。
|
||||
|
||||
6. **適合度分析** -- 各モデルはメモリ互換性について評価されます:
|
||||
|
||||
**実行モード:**
|
||||
- **GPU** -- モデルが VRAM に収まる。高速な推論。
|
||||
- **MoE** -- エキスパートオフロード付きの Mixture-of-Experts。アクティブなエキスパートは VRAM に、非アクティブは RAM に。
|
||||
- **CPU+GPU** -- VRAM が不足し、部分的な GPU オフロードでシステム RAM にあふれる。
|
||||
- **CPU** -- GPU なし。モデルは完全にシステム RAM に読み込まれる。
|
||||
|
||||
**適合レベル:**
|
||||
- **Perfect** -- GPU で推奨メモリを満たす。GPU アクセラレーションが必要。
|
||||
- **Good** -- 余裕を持って収まる。MoE オフロードや CPU+GPU で達成可能な最良。
|
||||
- **Marginal** -- ぎりぎりの適合、または CPU のみ（CPU のみは常にここで頭打ち）。
|
||||
- **Too Tight** -- どこにも十分な VRAM やシステム RAM がない。
|
||||
|
||||
---
|
||||
|
||||
## モデルデータベース
|
||||
|
||||
モデルリストは、HuggingFace REST API をクエリするスタンドアロンの Python スクリプト `scripts/scrape_hf_models.py`（標準ライブラリのみ、pip 依存なし）によって生成されます。Meta Llama、Mistral、Qwen、Google Gemma、Microsoft Phi、DeepSeek、IBM Granite、Allen Institute OLMo、xAI Grok、Cohere、BigCode、01.ai、Upstage、TII Falcon、HuggingFace、Zhipu GLM、Moonshot Kimi、Baidu ERNIE など、数百のモデルとプロバイダーを含みます。スクレイパーは、モデル設定（`num_local_experts`、`num_experts_per_tok`）と既知のアーキテクチャマッピングを通じて MoE アーキテクチャを自動検出します。
|
||||
|
||||
モデルカテゴリは、汎用、コーディング（CodeLlama、StarCoder2、WizardCoder、Qwen2.5-Coder、Qwen3-Coder）、推論（DeepSeek-R1、Orca-2）、マルチモーダル/ビジョン（Llama 3.2 Vision、Llama 4 Scout/Maverick、Qwen2.5-VL）、チャット、エンタープライズ（IBM Granite）、埋め込み（nomic-embed、bge）にわたります。
|
||||
|
||||
完全なリストは [MODELS.md](MODELS.md) を参照してください。
|
||||
|
||||
モデルデータベースはコンパイル時に埋め込まれるため、**エンドユーザー**は llmfit 自体をアップグレード（`brew upgrade llmfit`、`scoop update llmfit`、または新しいリリースのダウンロード）することで更新を受け取ります。以下のコマンドは、ソースからデータベースを更新する**コントリビューター**向けです:
|
||||
|
||||
モデルデータベースを更新するには:
|
||||
|
||||
```sh
|
||||
# 自動更新（推奨）
|
||||
make update-models
|
||||
|
||||
# またはスクリプトを直接実行
|
||||
./scripts/update_models.sh
|
||||
|
||||
# または手動で
|
||||
python3 scripts/scrape_hf_models.py
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
スクレイパーは `data/hf_models.json` を書き込み、これは `include_str!` を介してバイナリに焼き込まれます。自動更新スクリプトは既存データをバックアップし、JSON 出力を検証し、バイナリを再ビルドします。
|
||||
|
||||
デフォルトでは、スクレイパーは [unsloth](https://huggingface.co/unsloth) や [bartowski](https://huggingface.co/bartowski) などのプロバイダーから既知の GGUF ダウンロードソースでモデルを補強します。結果は `data/gguf_sources_cache.json` にキャッシュされ（7 日間の TTL）、API 呼び出しの繰り返しを避けます。補強をスキップしてスクレイプを高速化するには `--no-gguf-sources` を使ってください。
|
||||
|
||||
---
|
||||
|
||||
## プロジェクト構成
|
||||
|
||||
```
|
||||
src/
|
||||
main.rs -- CLI 引数解析、エントリポイント、TUI 起動
|
||||
hardware.rs -- システム RAM/CPU/GPU 検出（マルチ GPU、バックエンド識別）
|
||||
models.rs -- モデルデータベース、量子化階層、動的量子化選択
|
||||
fit.rs -- 多次元スコアリング（Q/S/F/C）、速度推定、MoE オフロード
|
||||
providers.rs -- ランタイムプロバイダー統合（Ollama、llama.cpp、MLX、Docker Model Runner、LM Studio）、インストール検出、pull/ダウンロード
|
||||
display.rs -- 従来型 CLI テーブルレンダリング + JSON 出力
|
||||
tui_app.rs -- TUI アプリケーション状態、フィルター、ナビゲーション
|
||||
tui_ui.rs -- TUI レンダリング（ratatui）
|
||||
tui_events.rs -- TUI キーボードイベント処理（crossterm）
|
||||
data/
|
||||
hf_models.json -- モデルデータベース（206 モデル）
|
||||
skills/
|
||||
llmfit-advisor/ -- ハードウェアを考慮したモデル推奨のための OpenClaw スキル
|
||||
scripts/
|
||||
scrape_hf_models.py -- HuggingFace API スクレイパー
|
||||
update_models.sh -- 自動データベース更新スクリプト
|
||||
install-openclaw-skill.sh -- OpenClaw スキルをインストール
|
||||
Makefile -- ビルドとメンテナンスのコマンド
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## crates.io への公開
|
||||
|
||||
`Cargo.toml` にはすでに必要なメタデータ（description、license、repository）が含まれています。公開するには:
|
||||
|
||||
```sh
|
||||
# 問題を検出するため、まずドライラン
|
||||
cargo publish --dry-run
|
||||
|
||||
# 本番の公開（crates.io API トークンが必要）
|
||||
cargo login
|
||||
cargo publish
|
||||
```
|
||||
|
||||
公開する前に、以下を確認してください:
|
||||
|
||||
- `Cargo.toml` のバージョンが正しいこと（リリースごとに上げる）。
|
||||
- リポジトリのルートに `LICENSE` ファイルが存在すること。なければ作成します:
|
||||
|
||||
```sh
|
||||
# MIT ライセンスの場合:
|
||||
curl -sL https://opensource.org/license/MIT -o LICENSE
|
||||
# または独自に書く。Cargo.toml は license = "MIT" を宣言しています。
|
||||
```
|
||||
|
||||
- `data/hf_models.json` がコミットされていること。これはコンパイル時に埋め込まれ、公開される crate に含まれている必要があります。
|
||||
|
||||
更新を公開するには:
|
||||
|
||||
```sh
|
||||
# バージョンを上げる
|
||||
# Cargo.toml を編集: version = "0.2.0"
|
||||
cargo publish
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 依存関係
|
||||
|
||||
| Crate | 目的 |
|
||||
|------------------------|--------------------------------------------------|
|
||||
| `clap` | derive マクロによる CLI 引数解析 |
|
||||
| `sysinfo` | クロスプラットフォームの RAM と CPU の検出 |
|
||||
| `serde` / `serde_json` | モデルデータベースの JSON デシリアライズ |
|
||||
| `tabled` | CLI テーブルフォーマット |
|
||||
| `colored` | CLI のカラー出力 |
|
||||
| `ureq` | ランタイム/プロバイダー API 統合のための HTTP クライアント |
|
||||
| `ratatui` | ターミナル UI フレームワーク |
|
||||
| `crossterm` | ratatui のためのターミナル入出力バックエンド |
|
||||
|
||||
---
|
||||
|
||||
## ランタイムプロバイダー統合
|
||||
|
||||
llmfit は複数のローカルランタイムプロバイダーをサポートします:
|
||||
|
||||
- **Ollama**（デーモン/API ベースの pull）
|
||||
- **llama.cpp**（Hugging Face からの直接 GGUF ダウンロード + ローカルキャッシュ検出）
|
||||
- **MLX**（Apple Silicon / mlx-community のモデルキャッシュ + オプションのサーバー） — MLX のダウンロードは元のモデル公開者ではなく、HuggingFace の `mlx-community/*` リポジトリにマッピングされます
|
||||
- **Docker Model Runner**（Docker Desktop 組み込みのモデルサービング）
|
||||
- **LM Studio**（モデル管理 + ダウンロード用の REST API を備えたローカルモデルサーバー）
|
||||
|
||||
あるモデルに対して互換性のあるプロバイダーが複数利用可能な場合、TUI で `d` を押すとプロバイダー選択モーダルが開きます。
|
||||
|
||||
### Ollama 統合
|
||||
|
||||
llmfit は [Ollama](https://ollama.com) と統合し、すでにインストールされているモデルを検出したり、TUI から直接新しいモデルをダウンロードしたりできます。
|
||||
|
||||
### 要件
|
||||
|
||||
- **Ollama がインストールされ実行中であること**（`ollama serve` または Ollama デスクトップアプリ）
|
||||
- llmfit は `http://localhost:11434`（Ollama のデフォルト API ポート）に接続します
|
||||
- 設定は不要 — Ollama が実行中なら、llmfit は自動的に検出します
|
||||
|
||||
### リモートの Ollama インスタンス
|
||||
|
||||
別のマシンやポートで実行中の Ollama に接続するには、`OLLAMA_HOST` 環境変数を設定します:
|
||||
|
||||
```sh
|
||||
# 特定の IP とポートの Ollama に接続
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit
|
||||
|
||||
# ホスト名経由で接続
|
||||
OLLAMA_HOST="http://ollama-server:666" llmfit
|
||||
|
||||
# すべての TUI および CLI コマンドで動作
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit --cli
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit fit --perfect -n 5
|
||||
```
|
||||
|
||||
これは以下の場合に便利です:
|
||||
- あるマシンで llmfit を実行し、Ollama を別のマシンからサーブする（例: GPU サーバー + ノート PC クライアント）
|
||||
- カスタムポートの Docker コンテナで実行中の Ollama に接続する
|
||||
- リバースプロキシやロードバランサーの背後にある Ollama を使う
|
||||
|
||||
### 仕組み
|
||||
|
||||
起動時、llmfit は `GET /api/tags` をクエリしてインストール済みの Ollama モデルを一覧表示します。インストール済みの各モデルには、TUI の **Inst** 列に緑の **✓** が付きます。システムバーには `Ollama: ✓ (N installed)` と表示されます。
|
||||
|
||||
モデルで `d` を押すと、llmfit は `POST /api/pull` を Ollama に送ってダウンロードします。行はアニメーション付きのプログレスインジケーターでハイライトされ、ダウンロードの進捗がリアルタイムで表示されます。完了すると、モデルはすぐに Ollama で利用可能になります。
|
||||
|
||||
Ollama が実行されていない場合、Ollama 固有の操作はスキップされます。TUI は利用可能であれば llama.cpp など他のプロバイダーを引き続きサポートします。
|
||||
|
||||
### llama.cpp 統合
|
||||
|
||||
llmfit は [llama.cpp](https://github.com/ggml-org/llama.cpp) を、TUI と CLI の両方でランタイム/ダウンロードプロバイダーとして統合します。
|
||||
|
||||
要件:
|
||||
|
||||
- `llama-cli` または `llama-server` が `PATH` で利用可能であること（ランタイム検出用）
|
||||
- GGUF ダウンロードのための Hugging Face へのネットワークアクセス
|
||||
|
||||
仕組み:
|
||||
|
||||
- llmfit は HF モデルを既知の GGUF リポジトリにマッピングします（ヒューリスティックなフォールバック付き）
|
||||
- GGUF ファイルをローカルの llama.cpp モデルキャッシュにダウンロードします
|
||||
- 一致する GGUF ファイルがローカルに存在する場合、モデルをインストール済みとしてマークします
|
||||
|
||||
#### 環境変数
|
||||
|
||||
| 変数 | デフォルト | 説明 |
|
||||
|---|---|---|
|
||||
| `LLAMA_CPP_PATH` | *(なし)* | llama.cpp バイナリ（`llama-cli`、`llama-server`）を含むディレクトリ。`PATH` ルックアップの前にチェックされます。 |
|
||||
| `LLAMA_SERVER_PORT` | `8080` | ランタイム検出のために実行中の `llama-server` のヘルスエンドポイントを探索する際に使用するポート。 |
|
||||
|
||||
llama.cpp が標準外の場所にインストールされている場合は、`PATH` に含めることを要求せずに llmfit が見つけられるよう、`LLAMA_CPP_PATH` を設定してください。
|
||||
|
||||
### Docker Model Runner 統合
|
||||
|
||||
llmfit は Docker Desktop 組み込みのモデルサービング機能である [Docker Model Runner](https://docs.docker.com/desktop/features/model-runner/) と統合します。
|
||||
|
||||
要件:
|
||||
|
||||
- Model Runner が有効になった Docker Desktop
|
||||
- デフォルトエンドポイント: `http://localhost:12434`
|
||||
|
||||
仕組み:
|
||||
|
||||
- llmfit は `GET /engines` をクエリして Docker Model Runner で利用可能なモデルを一覧表示します
|
||||
- モデルは Ollama スタイルのタグマッピングを使って HF データベースと照合されます（Docker Model Runner は `ai/<tag>` 命名を使用）
|
||||
- TUI で `d` を押すと `docker model pull` 経由で pull します
|
||||
|
||||
### リモートの Docker Model Runner インスタンス
|
||||
|
||||
別のホストやポートの Docker Model Runner に接続するには、`DOCKER_MODEL_RUNNER_HOST` 環境変数を設定します:
|
||||
|
||||
```sh
|
||||
DOCKER_MODEL_RUNNER_HOST="http://192.168.1.100:12434" llmfit
|
||||
```
|
||||
|
||||
### LM Studio 統合
|
||||
|
||||
llmfit は、組み込みのモデルダウンロード機能を備えたローカルモデルサーバーである [LM Studio](https://lmstudio.ai) と統合します。
|
||||
|
||||
要件:
|
||||
|
||||
- LM Studio がローカルサーバーを有効にして実行中であること
|
||||
- デフォルトエンドポイント: `http://127.0.0.1:1234`
|
||||
|
||||
仕組み:
|
||||
|
||||
- llmfit は `GET /v1/models` をクエリして LM Studio で利用可能なモデルを一覧表示します
|
||||
- TUI で `d` を押すと `POST /api/v1/models/download` 経由でダウンロードをトリガーします
|
||||
- ダウンロードの進捗は `GET /api/v1/models/download-status` をポーリングして追跡されます
|
||||
- LM Studio は HuggingFace のモデル名を直接受け入れるため、名前のマッピングは不要です
|
||||
|
||||
### リモートの LM Studio インスタンス
|
||||
|
||||
別のホストやポートの LM Studio に接続するには、`LMSTUDIO_HOST` 環境変数を設定します:
|
||||
|
||||
```sh
|
||||
LMSTUDIO_HOST="http://192.168.1.100:1234" llmfit
|
||||
```
|
||||
|
||||
### モデル名のマッピング
|
||||
|
||||
llmfit のデータベースは HuggingFace のモデル名（例: `Qwen/Qwen2.5-Coder-14B-Instruct`）を使いますが、Ollama は独自の命名スキーム（例: `qwen2.5-coder:14b`）を使います。llmfit は両者の間の正確なマッピングテーブルを維持し、インストール検出と pull が正しいモデルに解決されるようにします。各マッピングは厳密で、`qwen2.5-coder:14b` はベースの `qwen2.5:14b` ではなく Coder モデルにマッピングされます。
|
||||
|
||||
---
|
||||
|
||||
## プラットフォームサポート
|
||||
|
||||
- **Linux** -- フルサポート。GPU 検出は `nvidia-smi`（NVIDIA）、`rocm-smi`（AMD）、sysfs/`lspci`（Intel Arc）、`npu-smi`（Ascend）経由。
|
||||
- **macOS (Apple Silicon)** -- フルサポート。`system_profiler` 経由で統合メモリを検出。VRAM = システム RAM（共有プール）。モデルは Metal GPU アクセラレーション経由で実行。
|
||||
- **macOS (Intel)** -- RAM と CPU の検出が動作。`nvidia-smi` が利用可能ならディスクリート GPU を検出。
|
||||
- **Windows** -- RAM と CPU の検出が動作。インストールされていれば `nvidia-smi` 経由で NVIDIA GPU を検出。
|
||||
- **Android / Termux / PRoot** -- CPU と RAM の検出は通常動作しますが、GPU の自動検出は現在サポートされていません。Adreno などのモバイル GPU は、llmfit が使うデスクトップ/サーバーの探索インターフェースからは通常見えません。
|
||||
|
||||
### GPU サポート
|
||||
|
||||
| ベンダー | 検出方法 | VRAM レポート |
|
||||
|------------------------|-------------------------------|--------------------------------|
|
||||
| NVIDIA | `nvidia-smi` | 正確な専用 VRAM |
|
||||
| AMD | `rocm-smi` | 検出（VRAM は不明な場合あり） |
|
||||
| Intel Arc（ディスクリート） | sysfs（`mem_info_vram_total`） | 正確な専用 VRAM |
|
||||
| Intel Arc（統合） | `lspci` | 共有システムメモリ |
|
||||
| Apple Silicon | `system_profiler` | 統合メモリ（= システム RAM） |
|
||||
| Ascend | `npu-smi` | 検出（VRAM は不明な場合あり） |
|
||||
|
||||
自動検出が失敗したり誤った値を報告したりする場合は、`--memory`、`--ram`、`--cpu-cores` を使って上書きしてください（上記の[ハードウェアの上書き](#ハードウェアの上書き)を参照）。
|
||||
|
||||
### Android / Termux に関する注意
|
||||
|
||||
**Termux + PRoot** などの Android 構成では、llmfit は標準的な Linux 検出パス（`nvidia-smi`、`rocm-smi`、DRM/sysfs、`lspci` など）を通じてモバイル GPU を見ることが通常できません。そうした環境では、現在の実装では「GPU が検出されない」のが想定どおりの動作です。
|
||||
|
||||
それでも統合メモリのスマートフォンやタブレットで GPU スタイルの推奨が欲しい場合は、手動のメモリ上書きを使ってください:
|
||||
|
||||
```sh
|
||||
llmfit --memory=8G fit -n 20
|
||||
llmfit recommend --json --memory=8G --limit 10
|
||||
```
|
||||
|
||||
これは推奨/スコアリングのみのための回避策であり、真の Android GPU ランタイム検出を提供するものではありません。
|
||||
|
||||
---
|
||||
|
||||
## コントリビューション
|
||||
|
||||
コントリビューション、特に新しいモデルの追加を歓迎します。
|
||||
|
||||
### PR を提出する前に
|
||||
|
||||
変更をプッシュする前に `cargo fmt` を実行してください。ほとんどの CI チェックの失敗は、フォーマットされていないコードが原因です:
|
||||
|
||||
```sh
|
||||
cargo fmt
|
||||
```
|
||||
|
||||
### モデルの追加
|
||||
|
||||
1. モデルの HuggingFace リポジトリ ID（例: `meta-llama/Llama-3.1-8B`）を `scripts/scrape_hf_models.py` の `TARGET_MODELS` リストに追加します。
|
||||
2. モデルがゲートされている場合（メタデータへのアクセスに HuggingFace 認証が必要な場合）、同じスクリプトの `FALLBACKS` リストにパラメータ数とコンテキスト長を含むフォールバックエントリを追加します。
|
||||
3. 自動更新スクリプトを実行します:
|
||||
```sh
|
||||
make update-models
|
||||
# または: ./scripts/update_models.sh
|
||||
```
|
||||
4. 更新されたモデルリストを確認します: `./target/release/llmfit list`
|
||||
5. 次を実行して [MODELS.md](MODELS.md) を更新します: `python3 << 'EOF' < scripts/...`（ジェネレータースクリプトはコミット履歴を参照）
|
||||
6. プルリクエストを開きます。
|
||||
|
||||
現在のリストは [MODELS.md](MODELS.md)、アーキテクチャの詳細は [AGENTS.md](AGENTS.md) を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## OpenClaw 統合
|
||||
|
||||
llmfit は [OpenClaw](https://github.com/openclaw/openclaw) スキルとして提供され、エージェントがハードウェアに適したローカルモデルを推奨し、Ollama/vLLM/LM Studio プロバイダーを自動設定できるようにします。
|
||||
|
||||
### スキルのインストール
|
||||
|
||||
```sh
|
||||
# llmfit リポジトリから
|
||||
./scripts/install-openclaw-skill.sh
|
||||
|
||||
# または手動で
|
||||
cp -r skills/llmfit-advisor ~/.openclaw/skills/
|
||||
```
|
||||
|
||||
インストールしたら、OpenClaw エージェントに次のようなことを尋ねられます:
|
||||
|
||||
- 「どんなローカルモデルを実行できる?」
|
||||
- 「私のハードウェアに合うコーディングモデルを推奨して」
|
||||
- 「私の GPU に最適なモデルで Ollama をセットアップして」
|
||||
|
||||
エージェントは内部で `llmfit recommend --json` を呼び出し、結果を解釈して、最適なモデル選択で `openclaw.json` を設定することを提案します。
|
||||
|
||||
### 仕組み
|
||||
|
||||
このスキルは OpenClaw エージェントに次のことを教えます:
|
||||
|
||||
1. `llmfit --json system` でハードウェアを検出する
|
||||
2. `llmfit recommend --json` でランク付けされた推奨を取得する
|
||||
3. HuggingFace のモデル名を Ollama/vLLM/LM Studio のタグにマッピングする
|
||||
4. `openclaw.json` の `models.providers.ollama.models` を設定する
|
||||
|
||||
完全なスキル定義は [skills/llmfit-advisor/SKILL.md](skills/llmfit-advisor/SKILL.md) を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## 代替ツール
|
||||
|
||||
別のアプローチをお探しなら、[llm-checker](https://github.com/Pavelevich/llm-checker) をチェックしてください。これは Ollama 統合を備えた Node.js 製の CLI ツールで、モデルを直接 pull してベンチマークできます。スペックから推定するのではなく、実際に Ollama 経由でハードウェア上でモデルを動かすという、より実践的なアプローチを取ります。すでに Ollama がインストールされていて実環境のパフォーマンスをテストしたい場合に適しています。ただし MoE（Mixture-of-Experts）アーキテクチャはサポートしていない点に注意してください。すべてのモデルが密（dense）として扱われるため、Mixtral や DeepSeek-V3 のようなモデルのメモリ推定値は、より小さいアクティブな部分集合ではなく総パラメータ数を反映します。
|
||||
|
||||
---
|
||||
|
||||
## コード署名
|
||||
|
||||
llmfit の Windows リリースバイナリは [SignPath.io](https://about.signpath.io/) によりデジタル署名（Authenticode）されており、コード署名証明書は [SignPath Foundation](https://signpath.org/) から無償で提供されています。
|
||||
|
||||
署名は[リリースパイプライン](.github/workflows/release.yml)で自動的に行われます。署名に提出されるのは GitHub Actions によって本リポジトリからビルドされた成果物のみで、署名リクエストはプロジェクトメンテナー（[@AlexsJones](https://github.com/AlexsJones)）が承認します。
|
||||
|
||||
**コード署名ポリシー：**[SignPath Foundation のコード署名ポリシーと利用規約](https://signpath.org/terms)を参照してください。
|
||||
|
||||
**プライバシー：**本プログラムは、ユーザーまたは本プログラムをインストール・操作する人が明示的に要求しない限り、他のネットワークシステムへ情報を送信することはありません。llmfit が外部サービスにアクセスするのは、該当機能（モデルのダウンロード、ランタイムプロバイダーへの問い合わせ、コミュニティリーダーボードなど）を明示的に使用した場合のみです。
|
||||
|
||||
---
|
||||
|
||||
## ライセンス
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,190 @@
|
||||
# llmfit
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/icon.svg" alt="llmfit icon" width="128" height="128">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<b>English</b> ·
|
||||
<a href="README.zh.md">中文</a> ·
|
||||
<a href="README.ja.md">日本語</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/AlexsJones/llmfit/actions/workflows/ci.yml"><img src="https://github.com/AlexsJones/llmfit/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
||||
<a href="https://crates.io/crates/llmfit"><img src="https://img.shields.io/crates/v/llmfit.svg" alt="Crates.io"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License"></a>
|
||||
<a href="https://about.signpath.io"><img src="https://img.shields.io/badge/SignPath-signed-brightgreen?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxNiIgaGVpZ2h0PSIxNiIgZmlsbD0id2hpdGUiIHZpZXdCb3g9IjAgMCAxNiAxNiI+PHBhdGggZD0iTTEwLjA2NyA0LjU2N2wtNC43MzQgNC43MzMtMS40LTEuNGExIDEgMCAwIDAtMS40MTQgMS40MTRsMi4xIDIuMWExIDEgMCAwIDAgMS40MTQgMGw1LjQ0LTUuNDRhMSAxIDAgMCAwLTEuNDE0LTEuNDE0eiIvPjwvc3ZnPg==" alt="Signed with SignPath"></a>
|
||||
</p>
|
||||
|
||||
> **📊 New: benchmark & share — real numbers from your machine, better estimates for everyone.** `llmfit bench --share` measures real tok/s on your hardware and contributes it back to the project as a PR — no `gh` CLI, no third-party account. Every run is saved locally first (skip sharing, upload the backlog any time), your own measurements replace estimates in the fit table, and each merged submission ships in the next release: anyone on identical hardware gets measured `✓` numbers and calibrated estimates before they ever run a benchmark. [Get started with sharing →](docs/cli.md#contributing-benchmarks-bench---share)
|
||||
>
|
||||
> *Previously: [llmfit 1.0 — the release where the numbers became verifiable →](https://github.com/AlexsJones/llmfit/discussions/708)*
|
||||
|
||||
**Hundreds of models & providers. One command to find what runs on your hardware.**
|
||||
|
||||
A terminal tool that right-sizes LLM models to your system's RAM, CPU, and GPU. Detects your hardware, scores each model across quality, speed, fit, and context dimensions, and tells you which ones will actually run well on your machine.
|
||||
|
||||
Ships with an interactive TUI (default) and a classic CLI mode. Supports multi-GPU setups, MoE architectures, dynamic quantization selection, speed estimation, and local runtime providers (Ollama, llama.cpp, MLX, Docker Model Runner, LM Studio).
|
||||
|
||||
> **Sister projects:**
|
||||
> - [sympozium](https://github.com/sympozium-ai/sympozium/) — managing agents in Kubernetes.
|
||||
> - [llmserve](https://github.com/AlexsJones/llmserve) — a simple TUI for serving local LLM models. Pick a model, pick a backend, serve it.
|
||||
> - [llama-panel](https://github.com/AlexsJones/llama-panel) — a native macOS app for managing local llama-server instances.
|
||||
|
||||
|
||||
|
||||
## Documentation
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **Get started** | [Install](#install) · [Usage](#usage) · [How it works](#how-it-works) |
|
||||
| **Guides** | [TUI guide](docs/tui.md) · [CLI & automation](docs/cli.md) · [Runtime providers](docs/providers.md) · [OpenClaw integration](docs/openclaw.md) |
|
||||
| **Reference** | [How it works (full)](docs/how-it-works.md) · [Platform & GPU support](docs/platform-support.md) · [Custom models](docs/custom-models.md) · [Development](docs/development.md) |
|
||||
| **Project** | [Contributing](#contributing) · [Alternatives](#alternatives) · [Code signing](#code-signing) · [License](#license) |
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
### Windows
|
||||
```sh
|
||||
scoop install llmfit
|
||||
```
|
||||
|
||||
If Scoop is not installed, follow the [Scoop installation guide](https://scoop.sh/).
|
||||
|
||||
### macOS / Linux
|
||||
|
||||
#### Homebrew
|
||||
|
||||
Prebuilt binary (recommended, works on all macOS/Linux versions):
|
||||
```sh
|
||||
brew install AlexsJones/llmfit/llmfit
|
||||
```
|
||||
|
||||
Or from the homebrew-core formula, which builds from source on macOS versions without a bottle:
|
||||
```sh
|
||||
brew install llmfit
|
||||
```
|
||||
|
||||
#### MacPorts
|
||||
```sh
|
||||
port install llmfit
|
||||
```
|
||||
|
||||
#### Quick install
|
||||
```sh
|
||||
curl -fsSL https://llmfit.axjns.dev/install.sh | sh
|
||||
```
|
||||
|
||||
Downloads the latest release binary from GitHub and installs it to `/usr/local/bin` (or `~/.local/bin` if no sudo).
|
||||
|
||||
**Install to `~/.local/bin` without sudo:**
|
||||
```sh
|
||||
curl -fsSL https://llmfit.axjns.dev/install.sh | sh -s -- --local
|
||||
```
|
||||
|
||||
### uv / pip
|
||||
To install or update llmfit:
|
||||
```sh
|
||||
uv tool install -U llmfit
|
||||
```
|
||||
|
||||
To run without installing:
|
||||
```sh
|
||||
uvx llmfit
|
||||
```
|
||||
|
||||
You can also install llmfit as a Python package in the normal way with tools such as pip or uv.
|
||||
|
||||
### Docker / Podman
|
||||
```sh
|
||||
docker run ghcr.io/alexsjones/llmfit
|
||||
```
|
||||
This prints JSON from `llmfit recommend` command. The JSON could be further queried with `jq`.
|
||||
```
|
||||
podman run ghcr.io/alexsjones/llmfit recommend --use-case coding | jq '.models[].name'
|
||||
```
|
||||
To launch the interactive TUI instead, pass the global `--tui` flag:
|
||||
```sh
|
||||
docker run --rm -it ghcr.io/alexsjones/llmfit --tui
|
||||
```
|
||||
|
||||
### From source
|
||||
```sh
|
||||
git clone https://github.com/AlexsJones/llmfit.git
|
||||
cd llmfit
|
||||
cargo build --release
|
||||
# binary is at target/release/llmfit
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
llmfit # interactive TUI: your hardware, every model, ranked
|
||||
```
|
||||
|
||||
The TUI shows your detected specs at the top and every model scored for fit, speed, quality, and context. See the [TUI guide](docs/tui.md) for navigation, planning, simulation, downloads, the community leaderboard, and benchmarking.
|
||||
|
||||
For scripts, agents, and classic terminal output:
|
||||
|
||||
```sh
|
||||
llmfit fit # table of all models ranked by fit
|
||||
llmfit recommend --json # top picks as JSON (agent/script consumption)
|
||||
llmfit info "<model>" # one model: fit analysis, estimate basis, verify commands
|
||||
llmfit bench # measure real tok/s/TTFT against your running provider
|
||||
llmfit doctor # hardware detection report for bug reports
|
||||
```
|
||||
|
||||
Full reference: [CLI & automation](docs/cli.md).
|
||||
|
||||
---
|
||||
|
||||
## How it works
|
||||
|
||||
llmfit detects your hardware (RAM, CPU, GPU/VRAM, backend), then scores every model in its catalog across four dimensions: memory fit, estimated speed, quality, and context. Speed estimates come from a memory-bandwidth model grounded in runtime sampling and real community measurements — and every estimate ships its inputs, so `llmfit info` shows exactly what a number assumes and how to verify it on your machine.
|
||||
|
||||
Full detail, including the estimation formulas and the model database: [How llmfit works](docs/how-it-works.md).
|
||||
|
||||
---
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome, especially new models.
|
||||
|
||||
### Before submitting a PR
|
||||
|
||||
Please run `cargo fmt` before pushing your changes. Most CI check failures are caused by unformatted code:
|
||||
|
||||
```sh
|
||||
cargo fmt
|
||||
```
|
||||
|
||||
Guides for adding models — locally (no rebuild) or to the built-in catalog: [Custom models](docs/custom-models.md).
|
||||
|
||||
---
|
||||
|
||||
## Alternatives
|
||||
|
||||
If you're looking for a different approach, check out [llm-checker](https://github.com/Pavelevich/llm-checker) -- a Node.js CLI tool with Ollama integration that can pull and benchmark models directly. It takes a more hands-on approach by actually running models on your hardware via Ollama, rather than estimating from specs. Good if you already have Ollama installed and want to test real-world performance. Note that it doesn't support MoE (Mixture-of-Experts) architectures -- all models are treated as dense, so memory estimates for models like Mixtral or DeepSeek-V3 will reflect total parameter count rather than the smaller active subset.
|
||||
|
||||
---
|
||||
|
||||
## Code signing
|
||||
|
||||
llmfit's Windows release binaries are digitally signed (Authenticode) via [SignPath.io](https://about.signpath.io/), with a free code signing certificate provided by the [SignPath Foundation](https://signpath.org/).
|
||||
|
||||
Signing happens automatically in the [release pipeline](.github/workflows/release.yml): only artifacts built by GitHub Actions from this repository are submitted for signing, and signing requests are approved by the project maintainer ([@AlexsJones](https://github.com/AlexsJones)).
|
||||
|
||||
**Code signing policy:** see the [SignPath Foundation code signing policy and terms](https://signpath.org/terms).
|
||||
|
||||
**Privacy:** this program will not transfer any information to other networked systems unless specifically requested by the user or the person installing or operating it. llmfit only contacts external services when you explicitly use the corresponding feature (e.g. model downloads, runtime provider queries, or the community leaderboard).
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`AlexsJones/llmfit`
|
||||
- 原始仓库：https://github.com/AlexsJones/llmfit
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+1005
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 177 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 MiB |
@@ -0,0 +1,91 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" fill="none">
|
||||
<defs>
|
||||
<linearGradient id="chipGrad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#6366f1"/>
|
||||
<stop offset="100%" stop-color="#8b5cf6"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="brainGrad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#22d3ee"/>
|
||||
<stop offset="100%" stop-color="#a78bfa"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="bgGrad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#0f172a"/>
|
||||
<stop offset="100%" stop-color="#1e1b4b"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
|
||||
<!-- Background rounded square -->
|
||||
<rect x="16" y="16" width="480" height="480" rx="96" fill="url(#bgGrad)"/>
|
||||
|
||||
<!-- Chip body -->
|
||||
<rect x="136" y="136" width="240" height="240" rx="24" fill="url(#chipGrad)" opacity="0.9"/>
|
||||
|
||||
<!-- Chip pins — top -->
|
||||
<rect x="188" y="96" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="228" y="96" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="268" y="96" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="308" y="96" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
|
||||
<!-- Chip pins — bottom -->
|
||||
<rect x="188" y="360" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="228" y="360" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="268" y="360" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
<rect x="308" y="360" width="16" height="56" rx="4" fill="#818cf8"/>
|
||||
|
||||
<!-- Chip pins — left -->
|
||||
<rect x="96" y="188" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="96" y="228" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="96" y="268" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="96" y="308" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
|
||||
<!-- Chip pins — right -->
|
||||
<rect x="360" y="188" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="360" y="228" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="360" y="268" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
<rect x="360" y="308" width="56" height="16" rx="4" fill="#818cf8"/>
|
||||
|
||||
<!-- Inner chip die -->
|
||||
<rect x="176" y="176" width="160" height="160" rx="16" fill="#1e1b4b" opacity="0.6"/>
|
||||
|
||||
<!-- Brain / neural network inside the chip -->
|
||||
<!-- Central nodes -->
|
||||
<circle cx="226" cy="226" r="10" fill="url(#brainGrad)" opacity="0.9"/>
|
||||
<circle cx="286" cy="226" r="10" fill="url(#brainGrad)" opacity="0.9"/>
|
||||
<circle cx="256" cy="256" r="14" fill="url(#brainGrad)"/>
|
||||
<circle cx="226" cy="286" r="10" fill="url(#brainGrad)" opacity="0.9"/>
|
||||
<circle cx="286" cy="286" r="10" fill="url(#brainGrad)" opacity="0.9"/>
|
||||
|
||||
<!-- Outer nodes -->
|
||||
<circle cx="196" cy="256" r="7" fill="#22d3ee" opacity="0.7"/>
|
||||
<circle cx="316" cy="256" r="7" fill="#22d3ee" opacity="0.7"/>
|
||||
<circle cx="256" cy="196" r="7" fill="#22d3ee" opacity="0.7"/>
|
||||
<circle cx="256" cy="316" r="7" fill="#22d3ee" opacity="0.7"/>
|
||||
|
||||
<!-- Neural connections -->
|
||||
<line x1="226" y1="226" x2="286" y2="226" stroke="#22d3ee" stroke-width="2" opacity="0.5"/>
|
||||
<line x1="226" y1="286" x2="286" y2="286" stroke="#22d3ee" stroke-width="2" opacity="0.5"/>
|
||||
<line x1="226" y1="226" x2="226" y2="286" stroke="#22d3ee" stroke-width="2" opacity="0.5"/>
|
||||
<line x1="286" y1="226" x2="286" y2="286" stroke="#22d3ee" stroke-width="2" opacity="0.5"/>
|
||||
<line x1="226" y1="226" x2="256" y2="256" stroke="#a78bfa" stroke-width="2.5" opacity="0.7"/>
|
||||
<line x1="286" y1="226" x2="256" y2="256" stroke="#a78bfa" stroke-width="2.5" opacity="0.7"/>
|
||||
<line x1="226" y1="286" x2="256" y2="256" stroke="#a78bfa" stroke-width="2.5" opacity="0.7"/>
|
||||
<line x1="286" y1="286" x2="256" y2="256" stroke="#a78bfa" stroke-width="2.5" opacity="0.7"/>
|
||||
|
||||
<!-- Connections to outer nodes -->
|
||||
<line x1="196" y1="256" x2="226" y2="226" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="196" y1="256" x2="226" y2="286" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="316" y1="256" x2="286" y2="226" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="316" y1="256" x2="286" y2="286" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="256" y1="196" x2="226" y2="226" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="256" y1="196" x2="286" y2="226" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="256" y1="316" x2="226" y2="286" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
<line x1="256" y1="316" x2="286" y2="286" stroke="#22d3ee" stroke-width="1.5" opacity="0.4"/>
|
||||
|
||||
<!-- Glow on central node -->
|
||||
<circle cx="256" cy="256" r="22" fill="#22d3ee" opacity="0.12"/>
|
||||
|
||||
<!-- Checkmark overlay — bottom right -->
|
||||
<circle cx="380" cy="380" r="52" fill="#10b981"/>
|
||||
<circle cx="380" cy="380" r="46" fill="#059669"/>
|
||||
<polyline points="356,380 374,398 406,362" stroke="white" stroke-width="8" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 718 KiB |
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"id": "onnx-community/Phi-3-mini-4k-instruct",
|
||||
"name": "Phi-3-mini-4k-instruct",
|
||||
"parameters": "3.8B",
|
||||
"license": "mit",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 15300000000,
|
||||
"fp16": 7660000000,
|
||||
"q8": 3820000000,
|
||||
"int8": 3820000000,
|
||||
"uint8": 3820000000,
|
||||
"q4": 2390000000,
|
||||
"q4f16": 2440000000,
|
||||
"bnb4": 2280000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/Qwen2.5-0.5B-Instruct",
|
||||
"name": "Qwen2.5-0.5B-Instruct",
|
||||
"parameters": "0.5B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 1980000000,
|
||||
"fp16": 990000000,
|
||||
"q8": 530000000,
|
||||
"int8": 530000000,
|
||||
"uint8": 530000000,
|
||||
"q4": 380000000,
|
||||
"q4f16": 400000000,
|
||||
"bnb4": 360000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/Qwen2.5-1.5B-Instruct",
|
||||
"name": "Qwen2.5-1.5B-Instruct",
|
||||
"parameters": "1.5B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 6170000000,
|
||||
"fp16": 3090000000,
|
||||
"q8": 1620000000,
|
||||
"int8": 1620000000,
|
||||
"uint8": 1620000000,
|
||||
"q4": 990000000,
|
||||
"q4f16": 1020000000,
|
||||
"bnb4": 950000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/SmolLM2-1.7B-Instruct",
|
||||
"name": "SmolLM2-1.7B-Instruct",
|
||||
"parameters": "1.7B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 6840000000,
|
||||
"fp16": 3420000000,
|
||||
"q8": 1800000000,
|
||||
"int8": 1800000000,
|
||||
"uint8": 1800000000,
|
||||
"q4": 1090000000,
|
||||
"q4f16": 1120000000,
|
||||
"bnb4": 1040000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/gemma-2-2b-it-ONNX",
|
||||
"name": "gemma-2-2b-it",
|
||||
"parameters": "2.6B",
|
||||
"license": "gemma",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 10450000000,
|
||||
"fp16": 5230000000,
|
||||
"q8": 2750000000,
|
||||
"int8": 2750000000,
|
||||
"uint8": 2750000000,
|
||||
"q4": 1660000000,
|
||||
"q4f16": 1710000000,
|
||||
"bnb4": 1590000000
|
||||
}
|
||||
}
|
||||
]
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
# CLI & Automation
|
||||
|
||||
Classic table output, the REST API for cluster scheduling, hardware overrides, context caps, and JSON output for scripts and agents.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
### CLI mode
|
||||
|
||||
Use `--cli` or any subcommand to get classic table output:
|
||||
|
||||
```sh
|
||||
# Table of all models ranked by fit
|
||||
llmfit --cli
|
||||
|
||||
# Only perfectly fitting models, top 5
|
||||
llmfit fit --perfect -n 5
|
||||
|
||||
# Show detected system specs
|
||||
llmfit system
|
||||
|
||||
# Hardware diagnostic report for bug reports (raw nvidia-smi/rocm-smi/sysfs
|
||||
# output + what llmfit detected) — paste into a GitHub issue
|
||||
llmfit doctor
|
||||
|
||||
# List all models in the database
|
||||
llmfit list
|
||||
|
||||
# Search by name, provider, or size
|
||||
llmfit search "llama 8b"
|
||||
|
||||
# Detailed view of a single model
|
||||
llmfit info "Mistral-7B"
|
||||
|
||||
# Top 5 recommendations (JSON, for agent/script consumption)
|
||||
llmfit recommend --json --limit 5
|
||||
|
||||
# Recommendations filtered by use case
|
||||
llmfit recommend --json --use-case coding --limit 3
|
||||
|
||||
# Force a specific runtime (bypass automatic MLX selection on Apple Silicon)
|
||||
llmfit recommend --force-runtime llamacpp
|
||||
llmfit recommend --force-runtime llamacpp --use-case coding --limit 3
|
||||
|
||||
# Plan required hardware for a specific model configuration
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192 --quant mlx-4bit
|
||||
llmfit plan "Qwen/Qwen3-4B-MLX-4bit" --context 8192 --target-tps 25 --json
|
||||
|
||||
# Run as a node-level REST API (for cluster schedulers / aggregators)
|
||||
llmfit serve --host 0.0.0.0 --port 8787
|
||||
```
|
||||
|
||||
### REST API (`llmfit serve`)
|
||||
|
||||
`llmfit serve` starts an HTTP API that exposes the same fit/scoring data used by TUI/CLI, including filtering and top-model selection for a node.
|
||||
|
||||
```sh
|
||||
# Liveness
|
||||
curl http://localhost:8787/health
|
||||
|
||||
# Node hardware info
|
||||
curl http://localhost:8787/api/v1/system
|
||||
|
||||
# Full fit list with filters
|
||||
curl "http://localhost:8787/api/v1/models?min_fit=marginal&runtime=llamacpp&sort=score&limit=20"
|
||||
|
||||
# Key scheduling endpoint: top runnable models for this node
|
||||
curl "http://localhost:8787/api/v1/models/top?limit=5&min_fit=good&use_case=coding"
|
||||
|
||||
# Search by model name/provider text
|
||||
curl "http://localhost:8787/api/v1/models/Mistral?runtime=any"
|
||||
```
|
||||
|
||||
Supported query params for `models`/`models/top`:
|
||||
|
||||
- `limit` (or `n`): max number of rows returned
|
||||
- `perfect`: `true|false` (forces perfect-only when `true`)
|
||||
- `min_fit`: `perfect|good|marginal|too_tight`
|
||||
- `runtime`: `any|mlx|llamacpp`
|
||||
- `use_case`: `general|coding|reasoning|chat|multimodal|embedding`
|
||||
- `provider`: provider text filter (substring)
|
||||
- `search`: free-text filter across name/provider/size/use-case
|
||||
- `sort`: `score|tps|params|mem|ctx|date|use_case`
|
||||
- `include_too_tight`: include non-runnable rows (default `false` on `/top`, `true` on `/models`)
|
||||
- `max_context`: per-request context cap for memory estimation
|
||||
- `force_runtime`: `mlx|llamacpp|vllm` — override automatic runtime selection during analysis
|
||||
|
||||
Validate API behavior locally:
|
||||
|
||||
```sh
|
||||
# spawn server automatically and run endpoint/schema/filter assertions
|
||||
python3 scripts/test_api.py --spawn
|
||||
|
||||
# or test an already-running server
|
||||
python3 scripts/test_api.py --base-url http://127.0.0.1:8787
|
||||
```
|
||||
|
||||
### Contributing benchmarks (`bench --share`)
|
||||
|
||||
`llmfit bench` measures inference performance against a running provider
|
||||
(Ollama, vLLM, MLX, or llama-server). llama-server is auto-detected on port
|
||||
8080 via its `/props` endpoint (override with `LLAMA_SERVER_HOST` for a full
|
||||
URL, or `LLAMA_SERVER_PORT`), or select it explicitly with
|
||||
`--provider llamacpp`. Add `--share` to contribute your results back to the
|
||||
project as a pull request — **no `gh` CLI and no account on a third-party
|
||||
service required**:
|
||||
|
||||
```sh
|
||||
# Benchmark every discovered model and open a PR with the results
|
||||
llmfit bench --all --share
|
||||
|
||||
# Preview the exact JSON payloads without contacting GitHub
|
||||
llmfit bench --all --share --dry-run
|
||||
|
||||
# Skip the confirmation prompt (e.g. for automation)
|
||||
llmfit bench --all --share --yes
|
||||
|
||||
# Upload previously stored local benchmarks without benchmarking again
|
||||
llmfit bench --share
|
||||
```
|
||||
|
||||
**Every successful bench run is also saved locally** (under
|
||||
`~/.local/share/llmfit/benchmarks/pending/` on Linux; override the location
|
||||
with `LLMFIT_BENCH_STORE`), so skipping `--share` never discards data. These
|
||||
local results appear at the top of the TUI leaderboard as “you (local)”, and
|
||||
they feed back into the fit table: a model you benched shows your measured
|
||||
tok/s instead of the estimate, and runs on trustworthy models (≥ 1B params,
|
||||
dense) calibrate the formula estimates for **every other model** on the same
|
||||
hardware (shown as “Calibrated ×N from your own llmfit bench run(s)” in the
|
||||
estimate basis). Runs recorded on a different CPU/GPU configuration are
|
||||
ignored.
|
||||
Sharing later — `llmfit bench --share` on its own, or the share toggle in the
|
||||
TUI — offers to contribute **all** stored benchmarks in a single PR; uploaded
|
||||
files move to `.../benchmarks/shared/` so they are kept as history but never
|
||||
submitted twice.
|
||||
|
||||
**Merged submissions ship in the next release.** Community files are embedded
|
||||
into the binary at build time, so anyone on identical hardware (same CPU +
|
||||
GPU) sees them on the benchmark page as `llmfit community` rows, gets
|
||||
measured ✓ tok/s for those models, and gets calibrated estimates everywhere
|
||||
else — a fresh install benefits before its user ever runs a benchmark. Trust
|
||||
order everywhere: your own runs > llmfit community on identical hardware >
|
||||
localmaxxing medians on matching presets > formula estimate.
|
||||
|
||||
Authentication uses the GitHub **device flow** (the same mechanism
|
||||
`gh auth login` uses): llmfit prints a short code and a URL, you approve it in
|
||||
your browser once, and the token is cached under `~/.config/llmfit/` for next
|
||||
time. If a `GITHUB_TOKEN` or `GH_TOKEN` environment variable is set (or you use
|
||||
CI), that token is used automatically and no browser step is needed. With
|
||||
`--share`, credentials are resolved and verified **before** any benchmark
|
||||
starts, so a missing or expired token fails fast instead of after minutes of
|
||||
benching.
|
||||
|
||||
`--share` then forks the repo, commits one result file per stored submission
|
||||
under `llmfit-core/data/community/<hardware>/`, and opens a pull request — or,
|
||||
if you already have an open benchmark PR, **appends the new results to it**
|
||||
instead of opening another. Submissions are idempotent: file names mirror your
|
||||
local store, so retrying after a partial failure skips anything that already
|
||||
landed. Nothing is submitted until you confirm, and `--dry-run` never touches
|
||||
the network.
|
||||
|
||||
> Interactive login ships enabled — the public OAuth App client id is baked
|
||||
> into the binary (the device flow needs no client secret, so this is safe by
|
||||
> design). `LLMFIT_GH_CLIENT_ID` overrides it (e.g. when running a fork
|
||||
> against your own OAuth App); set it to an empty string to disable
|
||||
> interactive login entirely and rely on `GITHUB_TOKEN` / `GH_TOKEN`.
|
||||
|
||||
### Hardware overrides
|
||||
|
||||
Hardware autodetection can fail on some systems (e.g. broken `nvidia-smi`, VMs, passthrough setups), or you may want to evaluate model fit against different target hardware. Use `--memory`, `--ram`, and `--cpu-cores` to override detected values:
|
||||
|
||||
```sh
|
||||
# Override GPU VRAM
|
||||
llmfit --memory=32G
|
||||
|
||||
# Override system RAM
|
||||
llmfit --ram=128G
|
||||
|
||||
# Override CPU core count
|
||||
llmfit --cpu-cores=16
|
||||
|
||||
# Combine overrides to simulate target hardware
|
||||
llmfit --memory=24G --ram=64G --cpu-cores=8 fit
|
||||
llmfit --memory=24G --ram=64G system --json
|
||||
|
||||
# Works with all modes: TUI, CLI, and subcommands
|
||||
llmfit --memory=24G --cli
|
||||
llmfit --memory=24G fit --perfect -n 5
|
||||
llmfit --ram=64G recommend --json
|
||||
```
|
||||
|
||||
Accepted suffixes for `--memory` and `--ram`: `G`/`GB`/`GiB` (gigabytes), `M`/`MB`/`MiB` (megabytes), `T`/`TB`/`TiB` (terabytes). Case-insensitive. If no GPU was detected, `--memory` creates a synthetic GPU entry so models are scored for GPU inference. On unified-memory systems (Apple Silicon), `--ram` also updates VRAM; use `--memory` to override VRAM independently.
|
||||
|
||||
### Context-length cap for estimation
|
||||
|
||||
Use `--max-context` to cap context length used for memory estimation (without changing each model's advertised maximum context):
|
||||
|
||||
```sh
|
||||
# Estimate memory fit at 4K context
|
||||
llmfit --max-context 4096 --cli
|
||||
|
||||
# Works with subcommands
|
||||
llmfit --max-context 8192 fit --perfect -n 5
|
||||
llmfit --max-context 16384 recommend --json --limit 5
|
||||
```
|
||||
|
||||
If `--max-context` is not set, llmfit will use `OLLAMA_CONTEXT_LENGTH` when available.
|
||||
|
||||
### JSON output
|
||||
|
||||
Add `--json` to any subcommand for machine-readable output:
|
||||
|
||||
```sh
|
||||
llmfit --json system # Hardware specs as JSON
|
||||
llmfit --json fit -n 10 # Top 10 fits as JSON
|
||||
llmfit recommend --json # Top 5 recommendations (JSON is default for recommend)
|
||||
llmfit plan "Qwen/Qwen2.5-Coder-0.5B-Instruct" --context 8192 --json
|
||||
```
|
||||
|
||||
`plan` JSON includes stable fields for:
|
||||
- request (`context`, `quantization`, `target_tps`)
|
||||
- estimated minimum/recommended hardware
|
||||
- per-path feasibility (`gpu`, `cpu_offload`, `cpu_only`)
|
||||
- upgrade deltas
|
||||
|
||||
---
|
||||
@@ -0,0 +1,50 @@
|
||||
# Custom Models
|
||||
|
||||
Add your own models locally without a rebuild, or contribute a model to the built-in catalog.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
### Adding your own models locally (no rebuild needed)
|
||||
|
||||
You don't need to modify llmfit or wait for a release to see extra models. Create a `custom_models.json` in llmfit's data directory:
|
||||
|
||||
- Linux: `~/.local/share/llmfit/custom_models.json`
|
||||
- macOS: `~/Library/Application Support/llmfit/custom_models.json`
|
||||
- Windows: `%APPDATA%\llmfit\custom_models.json`
|
||||
|
||||
(or point the `LLMFIT_CUSTOM_MODELS` env var at any path). The file is a JSON array using the same entry format as the built-in catalog — see [llmfit-core/data/schema.json](../llmfit-core/data/schema.json); only a few fields are required:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "my-org/My-Model-7B",
|
||||
"provider": "my-org",
|
||||
"parameter_count": "7B",
|
||||
"min_ram_gb": 5.0,
|
||||
"recommended_ram_gb": 8.0,
|
||||
"min_vram_gb": 5.0,
|
||||
"quantization": "Q4_K_M",
|
||||
"context_length": 32768,
|
||||
"use_case": "General chat"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Custom entries with the same name as a catalog model **override** it; new names are added. Optional fields (`is_moe`, `num_hidden_layers`, `gguf_sources`, …) improve estimate accuracy when provided. You can also run `llmfit update` to fetch trending models from HuggingFace without a rebuild.
|
||||
|
||||
### Adding a model to the built-in catalog
|
||||
|
||||
1. Add the model's HuggingFace repo ID (e.g., `meta-llama/Llama-3.1-8B`) to the `TARGET_MODELS` list in `scripts/scrape_hf_models.py`.
|
||||
2. If the model is gated (requires HuggingFace authentication to access metadata), add a fallback entry to the `FALLBACKS` list in the same script with the parameter count and context length.
|
||||
3. Run the automated update script:
|
||||
```sh
|
||||
make update-models
|
||||
# or: ./scripts/update_models.sh
|
||||
```
|
||||
4. Verify the updated model list: `./target/release/llmfit list`
|
||||
5. Update [MODELS.md](../MODELS.md) by running: `python3 << 'EOF' < scripts/...` (see commit history for the generator script)
|
||||
6. Open a pull request.
|
||||
|
||||
See [MODELS.md](../MODELS.md) for the current list and [AGENTS.md](../AGENTS.md) for architecture details.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,82 @@
|
||||
# Development
|
||||
|
||||
Project structure, publishing, and dependencies.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
src/
|
||||
main.rs -- CLI argument parsing, entrypoint, TUI launch
|
||||
hardware.rs -- System RAM/CPU/GPU detection (multi-GPU, backend identification)
|
||||
models.rs -- Model database, quantization hierarchy, dynamic quant selection
|
||||
fit.rs -- Multi-dimensional scoring (Q/S/F/C), speed estimation, MoE offloading
|
||||
providers.rs -- Runtime provider integration (Ollama, llama.cpp, MLX, Docker Model Runner, LM Studio), install detection, pull/download
|
||||
display.rs -- Classic CLI table rendering + JSON output
|
||||
tui_app.rs -- TUI application state, filters, navigation
|
||||
tui_ui.rs -- TUI rendering (ratatui)
|
||||
tui_events.rs -- TUI keyboard event handling (crossterm)
|
||||
llmfit-core/data/
|
||||
hf_models.json -- Model database (embedded at compile time)
|
||||
skills/
|
||||
llmfit-advisor/ -- OpenClaw skill for hardware-aware model recommendations
|
||||
scripts/
|
||||
scrape_hf_models.py -- HuggingFace API scraper
|
||||
update_models.sh -- Automated database update script
|
||||
install-openclaw-skill.sh -- Install the OpenClaw skill
|
||||
Makefile -- Build and maintenance commands
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Publishing to crates.io
|
||||
|
||||
The `Cargo.toml` already includes the required metadata (description, license, repository). To publish:
|
||||
|
||||
```sh
|
||||
# Dry run first to catch issues
|
||||
cargo publish --dry-run
|
||||
|
||||
# Publish for real (requires a crates.io API token)
|
||||
cargo login
|
||||
cargo publish
|
||||
```
|
||||
|
||||
Before publishing, make sure:
|
||||
|
||||
- The version in `Cargo.toml` is correct (bump with each release).
|
||||
- A `LICENSE` file exists in the repo root. Create one if missing:
|
||||
|
||||
```sh
|
||||
# For MIT license:
|
||||
curl -sL https://opensource.org/license/MIT -o LICENSE
|
||||
# Or write your own. The Cargo.toml declares license = "MIT".
|
||||
```
|
||||
|
||||
- `llmfit-core/data/hf_models.json` is committed. It is embedded at compile time and must be present in the published crate.
|
||||
|
||||
To publish updates:
|
||||
|
||||
```sh
|
||||
# Bump version
|
||||
# Edit Cargo.toml: version = "0.2.0"
|
||||
cargo publish
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Crate | Purpose |
|
||||
|------------------------|--------------------------------------------------|
|
||||
| `clap` | CLI argument parsing with derive macros |
|
||||
| `sysinfo` | Cross-platform RAM and CPU detection |
|
||||
| `serde` / `serde_json` | JSON deserialization for model database |
|
||||
| `tabled` | CLI table formatting |
|
||||
| `colored` | CLI colored output |
|
||||
| `ureq` | HTTP client for runtime/provider API integration |
|
||||
| `ratatui` | Terminal UI framework |
|
||||
| `crossterm` | Terminal input/output backend for ratatui |
|
||||
|
||||
---
|
||||
@@ -0,0 +1,102 @@
|
||||
# How llmfit Works
|
||||
|
||||
The scoring model, speed estimation, and the model database.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
## How it works
|
||||
|
||||
1. **Hardware detection** -- Reads total/available RAM via `sysinfo`, counts CPU cores, and probes for GPUs:
|
||||
- **NVIDIA** -- Multi-GPU support via `nvidia-smi`. Aggregates VRAM across all detected GPUs. Falls back to VRAM estimation from GPU model name if reporting fails.
|
||||
- **AMD** -- Detected via `rocm-smi`.
|
||||
- **Intel Arc** -- Discrete VRAM via sysfs, integrated via `lspci`.
|
||||
- **Apple Silicon** -- Unified memory via `system_profiler`. VRAM = system RAM.
|
||||
- **Ascend** -- Detected via `npu-smi`.
|
||||
- **Backend detection** -- Automatically identifies the acceleration backend (CUDA, Metal, ROCm, SYCL, CPU ARM, CPU x86, Ascend) for speed estimation.
|
||||
|
||||
2. **Model database** -- Hundreds models sourced from the HuggingFace API, stored in `llmfit-core/data/hf_models.json` and embedded at compile time. Memory requirements are computed from parameter counts across a quantization hierarchy (Q8_0 through Q2_K). VRAM is the primary constraint for GPU inference; system RAM is the fallback for CPU-only execution.
|
||||
|
||||
**MoE support** -- Models with Mixture-of-Experts architectures (Mixtral, DeepSeek-V2/V3) are detected automatically. Only a subset of experts is active per token, so the effective VRAM requirement is much lower than total parameter count suggests. For example, Mixtral 8x7B has 46.7B total parameters but only activates ~12.9B per token, reducing VRAM from 23.9 GB to ~6.6 GB with expert offloading.
|
||||
|
||||
3. **Dynamic quantization** -- Instead of assuming a fixed quantization, llmfit tries the best quality quantization that fits your hardware. It walks a hierarchy from Q8_0 (best quality) down to Q2_K (most compressed), picking the highest quality that fits in available memory. If nothing fits at full context, it tries again at half context.
|
||||
|
||||
4. **Multi-dimensional scoring** -- Each model is scored across four dimensions (0–100 each):
|
||||
|
||||
| Dimension | What it measures |
|
||||
|-------------|--------------------------------------------------------------------------------|
|
||||
| **Quality** | Parameter count, model family reputation, quantization penalty, task alignment |
|
||||
| **Speed** | Estimated tokens/sec based on backend, params, and quantization |
|
||||
| **Fit** | Memory utilization efficiency (sweet spot: 50–80% of available memory) |
|
||||
| **Context** | Context window capability vs target for the use case |
|
||||
|
||||
Dimensions are combined into a weighted composite score. Weights vary by use-case category (General, Coding, Reasoning, Chat, Multimodal, Embedding). For example, Chat weights Speed higher (0.35) while Reasoning weights Quality higher (0.55). Models are ranked by composite score, with unrunnable models (Too Tight) always at the bottom.
|
||||
|
||||
Task alignment within the Quality dimension uses a curated per-family benchmark table ([llmfit-core/data/use_case_benchmarks.json](../llmfit-core/data/use_case_benchmarks.json), aggregated from public coding/reasoning/chat leaderboards), so a strong coding model outranks a larger generalist for `--use-case coding` even at fewer parameters. Families without an entry fall back to name-based heuristics; corrections to the table are welcome PRs.
|
||||
|
||||
5. **Speed estimation** -- Token generation in LLM inference is memory-bandwidth-bound: each token requires reading the full model weights once from VRAM. When the GPU model is recognized, llmfit uses its actual memory bandwidth to estimate throughput:
|
||||
|
||||
Formula: `(bandwidth_GB_s / model_size_GB) × efficiency_factor`
|
||||
|
||||
The efficiency factor (0.55) and per-mode speed multipliers are tunable via the Advanced Configuration popup (`A` in the TUI). The defaults account for kernel overhead, KV-cache reads, and memory controller effects. This approach is validated against published benchmarks from llama.cpp ([Apple Silicon](https://github.com/ggml-org/llama.cpp/discussions/4167), [NVIDIA T4](https://github.com/ggml-org/llama.cpp/discussions/4225)) and real-world measurements.
|
||||
|
||||
The bandwidth lookup table covers ~80 GPUs across NVIDIA (consumer + datacenter), AMD (RDNA + CDNA), and Apple Silicon families.
|
||||
|
||||
For unrecognized GPUs, llmfit falls back to per-backend speed constants:
|
||||
|
||||
| Backend | Speed constant |
|
||||
|--------------|----------------|
|
||||
| CUDA | 220 |
|
||||
| Metal | 160 |
|
||||
| ROCm | 180 |
|
||||
| SYCL | 100 |
|
||||
| CPU (ARM) | 90 |
|
||||
| CPU (x86) | 70 |
|
||||
| NPU (Ascend) | 390 |
|
||||
|
||||
Fallback formula: `K / params_b × quant_speed_multiplier`, with per-mode penalties tunable via the Advanced Configuration popup (`A` in the TUI).
|
||||
|
||||
6. **Fit analysis** -- Each model is evaluated for memory compatibility:
|
||||
|
||||
**Run modes:**
|
||||
- **GPU** -- Model fits in VRAM. Fast inference.
|
||||
- **MoE** -- Mixture-of-Experts with expert offloading. Active experts in VRAM, inactive in RAM.
|
||||
- **CPU+GPU** -- VRAM insufficient, spills to system RAM with partial GPU offload.
|
||||
- **CPU** -- No GPU. Model loaded entirely into system RAM.
|
||||
|
||||
**Fit levels:**
|
||||
- **Perfect** -- Recommended memory met on GPU. Requires GPU acceleration.
|
||||
- **Good** -- Fits with headroom. Best achievable for MoE offload or CPU+GPU.
|
||||
- **Marginal** -- Tight fit, or CPU-only (CPU-only always caps here).
|
||||
- **Too Tight** -- Not enough VRAM or system RAM anywhere.
|
||||
|
||||
---
|
||||
|
||||
## Model database
|
||||
|
||||
The model list is generated by `scripts/scrape_hf_models.py`, a standalone Python script (stdlib only, no pip dependencies) that queries the HuggingFace REST API. Hundreds models & providers including Meta Llama, Mistral, Qwen, Google Gemma, Microsoft Phi, DeepSeek, IBM Granite, Allen Institute OLMo, xAI Grok, Cohere, BigCode, 01.ai, Upstage, TII Falcon, HuggingFace, Zhipu GLM, Moonshot Kimi, Baidu ERNIE, and more. The scraper automatically detects MoE architectures via model config (`num_local_experts`, `num_experts_per_tok`) and known architecture mappings.
|
||||
|
||||
Model categories span general purpose, coding (CodeLlama, StarCoder2, WizardCoder, Qwen2.5-Coder, Qwen3-Coder), reasoning (DeepSeek-R1, Orca-2), multimodal/vision (Llama 3.2 Vision, Llama 4 Scout/Maverick, Qwen2.5-VL), chat, enterprise (IBM Granite), and embedding (nomic-embed, bge).
|
||||
|
||||
See [MODELS.md](../MODELS.md) for the full list.
|
||||
|
||||
The model database is embedded at compile time, so **end users** get updates by upgrading llmfit itself (`brew upgrade llmfit`, `scoop update llmfit`, or downloading a newer release). The commands below are for **contributors** refreshing the database from source:
|
||||
|
||||
To refresh the model database:
|
||||
|
||||
```sh
|
||||
# Automated update (recommended)
|
||||
make update-models
|
||||
|
||||
# Or run the script directly
|
||||
./scripts/update_models.sh
|
||||
|
||||
# Or manually
|
||||
python3 scripts/scrape_hf_models.py
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
The scraper writes `llmfit-core/data/hf_models.json`, which is baked into the binary via `include_str!`. The automated update script backs up existing data, validates JSON output, and rebuilds the binary.
|
||||
|
||||
By default, the scraper enriches models with known GGUF download sources from providers like [unsloth](https://huggingface.co/unsloth) and [bartowski](https://huggingface.co/bartowski). Results are cached in `data/gguf_sources_cache.json` (7-day TTL) to avoid repeated API calls. Use `--no-gguf-sources` to skip enrichment for a faster scrape.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,38 @@
|
||||
# OpenClaw Integration
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
## OpenClaw integration
|
||||
|
||||
llmfit ships as an [OpenClaw](https://github.com/openclaw/openclaw) skill that lets the agent recommend hardware-appropriate local models and auto-configure Ollama/vLLM/LM Studio providers.
|
||||
|
||||
### Install the skill
|
||||
|
||||
```sh
|
||||
# From the llmfit repo
|
||||
./scripts/install-openclaw-skill.sh
|
||||
|
||||
# Or manually
|
||||
cp -r skills/llmfit-advisor ~/.openclaw/skills/
|
||||
```
|
||||
|
||||
Once installed, ask your OpenClaw agent things like:
|
||||
|
||||
- "What local models can I run?"
|
||||
- "Recommend a coding model for my hardware"
|
||||
- "Set up Ollama with the best models for my GPU"
|
||||
|
||||
The agent will call `llmfit recommend --json` under the hood, interpret the results, and offer to configure your `openclaw.json` with optimal model choices.
|
||||
|
||||
### How it works
|
||||
|
||||
The skill teaches the OpenClaw agent to:
|
||||
|
||||
1. Detect your hardware via `llmfit --json system`
|
||||
2. Get ranked recommendations via `llmfit recommend --json`
|
||||
3. Map HuggingFace model names to Ollama/vLLM/LM Studio tags
|
||||
4. Configure `models.providers.ollama.models` in `openclaw.json`
|
||||
|
||||
See [skills/llmfit-advisor/SKILL.md](../skills/llmfit-advisor/SKILL.md) for the full skill definition.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,39 @@
|
||||
# Platform Support
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
## Platform support
|
||||
|
||||
- **Linux** -- Full support. GPU detection via `nvidia-smi` (NVIDIA), `rocm-smi` (AMD), sysfs/`lspci` (Intel Arc) and `npu-smi` (Ascend).
|
||||
- **macOS (Apple Silicon)** -- Full support. Detects unified memory via `system_profiler`. VRAM = system RAM (shared pool). Models run via Metal GPU acceleration.
|
||||
- **macOS (Intel)** -- RAM and CPU detection works. Discrete GPU detection if `nvidia-smi` available.
|
||||
- **Windows** -- RAM and CPU detection works. NVIDIA GPU detection via `nvidia-smi` if installed.
|
||||
- **Android / Termux / PRoot** -- CPU and RAM detection usually work, but GPU autodetection is not currently supported. Mobile GPUs such as Adreno typically are not visible through the desktop/server probing interfaces llmfit uses.
|
||||
|
||||
### GPU support
|
||||
|
||||
| Vendor | Detection method | VRAM reporting |
|
||||
|------------------------|-------------------------------|--------------------------------|
|
||||
| NVIDIA | `nvidia-smi` | Exact dedicated VRAM |
|
||||
| AMD | `rocm-smi` | Detected (VRAM may be unknown) |
|
||||
| Intel Arc (discrete) | sysfs (`mem_info_vram_total`) | Exact dedicated VRAM |
|
||||
| Intel Arc (integrated) | `lspci` | Shared system memory |
|
||||
| Apple Silicon | `system_profiler` | Unified memory (= system RAM) |
|
||||
| Ascend | `npu-smi` | Detected (VRAM may be unknown) |
|
||||
|
||||
If autodetection fails or reports incorrect values, use `--memory`, `--ram`, or `--cpu-cores` to override (see [Hardware overrides](cli.md#hardware-overrides)).
|
||||
|
||||
### Android / Termux note
|
||||
|
||||
On Android setups such as **Termux + PRoot**, llmfit usually cannot see mobile GPUs through the standard Linux detection paths (`nvidia-smi`, `rocm-smi`, DRM/sysfs, `lspci`, etc.). In those environments, "no GPU detected" is expected with the current implementation.
|
||||
|
||||
If you still want GPU-style recommendations on a unified-memory phone or tablet, use a manual memory override:
|
||||
|
||||
```sh
|
||||
llmfit --memory=8G fit -n 20
|
||||
llmfit recommend --json --memory=8G --limit 10
|
||||
```
|
||||
|
||||
This is a workaround for recommendation/scoring only; it does not provide true Android GPU runtime detection.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,142 @@
|
||||
# Runtime Provider Integration
|
||||
|
||||
How llmfit detects and talks to Ollama, llama.cpp, Docker Model Runner, LM Studio, and remote instances.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
## Runtime provider integration
|
||||
|
||||
llmfit supports multiple local runtime providers:
|
||||
|
||||
- **Ollama** (daemon/API based pulls)
|
||||
- **llama.cpp** (direct GGUF downloads from Hugging Face + local cache detection)
|
||||
- **MLX** (Apple Silicon / mlx-community model cache + optional server) — MLX downloads map to `mlx-community/*` repos on HuggingFace, not the original model publisher
|
||||
- **Docker Model Runner** (Docker Desktop's built-in model serving)
|
||||
- **LM Studio** (local model server with REST API for model management + downloads)
|
||||
|
||||
When more than one compatible provider is available for a model, pressing `d` in the TUI opens a provider picker modal.
|
||||
|
||||
### Ollama integration
|
||||
|
||||
llmfit integrates with [Ollama](https://ollama.com) to detect which models you already have installed and to download new ones directly from the TUI.
|
||||
|
||||
### Requirements
|
||||
|
||||
- **Ollama must be installed and running** (`ollama serve` or the Ollama desktop app)
|
||||
- llmfit connects to `http://localhost:11434` (Ollama's default API port)
|
||||
- No configuration needed — if Ollama is running, llmfit detects it automatically
|
||||
|
||||
### Remote Ollama instances
|
||||
|
||||
To connect to Ollama running on a different machine or port, set the `OLLAMA_HOST` environment variable:
|
||||
|
||||
```sh
|
||||
# Connect to Ollama on a specific IP and port
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit
|
||||
|
||||
# Connect via hostname
|
||||
OLLAMA_HOST="http://ollama-server:666" llmfit
|
||||
|
||||
# Works with all TUI and CLI commands
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit --cli
|
||||
OLLAMA_HOST="http://192.168.1.100:11434" llmfit fit --perfect -n 5
|
||||
```
|
||||
|
||||
This is useful for:
|
||||
- Running llmfit on one machine while Ollama serves from another (e.g., GPU server + laptop client)
|
||||
- Connecting to Ollama running in Docker containers with custom ports
|
||||
- Using Ollama behind reverse proxies or load balancers
|
||||
|
||||
### How it works
|
||||
|
||||
On startup, llmfit queries `GET /api/tags` to list your installed Ollama models. Each installed model gets a green **✓** in the **Inst** column of the TUI. The system bar shows `Ollama: ✓ (N installed)`.
|
||||
|
||||
When you press `d` on a model, llmfit sends `POST /api/pull` to Ollama to download it. The row highlights with an animated progress indicator showing download progress in real-time. Once complete, the model is immediately available for use with Ollama.
|
||||
|
||||
If Ollama is not running, Ollama-specific operations are skipped; the TUI still supports other providers like llama.cpp where available.
|
||||
|
||||
### llama.cpp integration
|
||||
|
||||
llmfit integrates with [llama.cpp](https://github.com/ggml-org/llama.cpp) as a runtime/download provider in both TUI and CLI.
|
||||
|
||||
Requirements:
|
||||
|
||||
- `llama-cli` or `llama-server` available in `PATH` (for runtime detection)
|
||||
- network access to Hugging Face for GGUF downloads
|
||||
|
||||
How it works:
|
||||
|
||||
- llmfit maps HF models to known GGUF repos (with heuristic fallbacks)
|
||||
- downloads GGUF files into the local llama.cpp model cache
|
||||
- marks models installed when matching GGUF files are present locally
|
||||
|
||||
#### Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `LLAMA_CPP_PATH` | *(none)* | Directory containing llama.cpp binaries (`llama-cli`, `llama-server`). Checked before `PATH` lookup. |
|
||||
| `LLAMA_SERVER_PORT` | `8080` | Port used when probing a running `llama-server` health endpoint for runtime detection. |
|
||||
|
||||
If llama.cpp is installed in a non-standard location, set `LLAMA_CPP_PATH` so llmfit can find it without requiring it in your `PATH`.
|
||||
|
||||
### Docker Model Runner integration
|
||||
|
||||
llmfit integrates with [Docker Model Runner](https://docs.docker.com/desktop/features/model-runner/), Docker Desktop's built-in model serving feature.
|
||||
|
||||
Requirements:
|
||||
|
||||
- Docker Desktop with Model Runner enabled
|
||||
- Default endpoint: `http://localhost:12434`
|
||||
|
||||
How it works:
|
||||
|
||||
- llmfit queries `GET /engines` to list models available in Docker Model Runner
|
||||
- models are matched to the HF database using Ollama-style tag mapping (Docker Model Runner uses `ai/<tag>` naming)
|
||||
- pressing `d` in the TUI pulls via `docker model pull`
|
||||
|
||||
### Remote Docker Model Runner instances
|
||||
|
||||
To connect to Docker Model Runner on a different host or port, set the `DOCKER_MODEL_RUNNER_HOST` environment variable:
|
||||
|
||||
```sh
|
||||
DOCKER_MODEL_RUNNER_HOST="http://192.168.1.100:12434" llmfit
|
||||
```
|
||||
|
||||
### LM Studio integration
|
||||
|
||||
llmfit integrates with [LM Studio](https://lmstudio.ai) as a local model server with built-in model download capabilities.
|
||||
|
||||
Requirements:
|
||||
|
||||
- LM Studio must be running with its local server enabled
|
||||
- Default endpoint: `http://127.0.0.1:1234`
|
||||
|
||||
How it works:
|
||||
|
||||
- llmfit queries `GET /v1/models` to list models available in LM Studio
|
||||
- pressing `d` in the TUI triggers a download via `POST /api/v1/models/download`
|
||||
- download progress is tracked by polling `GET /api/v1/models/download-status`
|
||||
- LM Studio accepts HuggingFace model names directly, so no name mapping is needed
|
||||
|
||||
### Remote LM Studio instances
|
||||
|
||||
To connect to LM Studio on a different host or port, set the `LMSTUDIO_HOST` environment variable:
|
||||
|
||||
```sh
|
||||
LMSTUDIO_HOST="http://192.168.1.100:1234" llmfit
|
||||
```
|
||||
|
||||
### API authentication
|
||||
|
||||
If your LM Studio instance has **Require API Key** enabled (required for MCP server access), set the `LMSTUDIO_API_KEY` environment variable to provide a Bearer token with all requests:
|
||||
|
||||
```sh
|
||||
export LMSTUDIO_API_KEY="your-api-key-here"
|
||||
llmfit
|
||||
```
|
||||
|
||||
### Model name mapping
|
||||
|
||||
llmfit's database uses HuggingFace model names (e.g. `Qwen/Qwen2.5-Coder-14B-Instruct`) while Ollama uses its own naming scheme (e.g. `qwen2.5-coder:14b`). llmfit maintains an accurate mapping table between the two so that install detection and pulls resolve to the correct model. Each mapping is exact — `qwen2.5-coder:14b` maps to the Coder model, not the base `qwen2.5:14b`.
|
||||
|
||||
---
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
# TUI Guide
|
||||
|
||||
Everything about llmfit's interactive terminal UI: navigation, planning, simulation, configuration, downloads, the community leaderboard, benchmarking, themes, and the web dashboard.
|
||||
|
||||
[← Back to README](../README.md)
|
||||
|
||||
### TUI (default)
|
||||
|
||||
```sh
|
||||
llmfit
|
||||
```
|
||||
|
||||
Launches the interactive terminal UI. Your system specs (CPU, RAM, GPU name, VRAM, backend) are shown at the top. Models are listed in a scrollable table sorted by composite score. Each row shows the model's score, estimated tok/s, best quantization for your hardware, run mode, memory usage, and use-case category.
|
||||
|
||||
| Key | Action |
|
||||
|----------------------------|-----------------------------------------------------------------------|
|
||||
| `Up` / `Down` or `j` / `k` | Navigate models |
|
||||
| `/` | Enter search mode (partial match on name, provider, params, use case) |
|
||||
| `Esc` or `Enter` | Exit search mode |
|
||||
| `Ctrl-U` | Clear search |
|
||||
| `f` | Cycle fit filter: All, Runnable, Perfect, Good, Marginal |
|
||||
| `a` | Cycle availability filter: All, GGUF Avail, Installed |
|
||||
| `s` | Cycle sort column: Score, Params, Mem%, Ctx, Date, Use Case |
|
||||
| `v` | Enter Visual mode (select multiple models) |
|
||||
| `V` | Enter Select mode (column-based filtering) |
|
||||
| `t` | Cycle color theme (saved automatically) |
|
||||
| `p` | Open Plan mode for selected model (hardware planning) |
|
||||
| `P` | Open provider filter popup (type to fuzzy-filter providers) |
|
||||
| `U` | Open use-case filter popup |
|
||||
| `C` | Open capability filter popup |
|
||||
| `L` | Open license filter popup |
|
||||
| `R` | Open runtime/backend filter popup (llama.cpp, MLX, vLLM) |
|
||||
| `S` | Open hardware simulation popup (override RAM/VRAM/CPU) |
|
||||
| `A` | Open advanced configuration popup (tune efficiency, run mode factors) |
|
||||
| `b` | Open community leaderboard view (localmaxxing.com) |
|
||||
| `I` | Open inference bench view (local quality scoring against your models) |
|
||||
| `h` | Open help popup (all key bindings) |
|
||||
| `m` | Mark selected model for compare |
|
||||
| `c` | Open compare view (marked vs selected) |
|
||||
| `x` | Clear compare mark |
|
||||
| `i` | Toggle installed-first sorting (any detected runtime provider) |
|
||||
| `d` | Download selected model (provider picker when multiple are available) |
|
||||
| `D` | Open Download Manager (history, deletion, config) |
|
||||
| `r` | Refresh installed models from runtime providers |
|
||||
| `Enter` | Toggle detail view for selected model |
|
||||
| `PgUp` / `PgDn` | Scroll by 10 |
|
||||
| `g` / `G` | Jump to top / bottom |
|
||||
| `q` | Quit |
|
||||
|
||||
### Vim-like modes
|
||||
|
||||
The TUI uses Vim-inspired modes shown in the bottom-left status bar. The current mode determines which keys are active.
|
||||
|
||||
#### Normal mode
|
||||
|
||||
The default mode. Navigate, search, filter, and open views. All keys in the table above apply here.
|
||||
|
||||
#### Visual mode (`v`)
|
||||
|
||||
Select a contiguous range of models for bulk comparison. Press `v` to anchor at the current row, then navigate with `j`/`k` or arrow keys to extend the selection. Selected rows are highlighted.
|
||||
|
||||
| Key | Action |
|
||||
|---------------------|--------------------------------------------------------|
|
||||
| `j` / `k` or arrows | Extend selection up/down |
|
||||
| `c` | Compare all selected models (opens multi-compare view) |
|
||||
| `m` | Mark current model for two-model compare |
|
||||
| `Esc` or `v` | Exit Visual mode |
|
||||
|
||||
The multi-compare view displays a table where rows are attributes (Score, tok/s, Fit, Mem%, Params, Mode, Context, Quant, etc.) and columns are models. Best values are highlighted. Use `h`/`l` or arrow keys to scroll horizontally if more models are selected than fit on screen.
|
||||
|
||||
#### Select mode (`V`)
|
||||
|
||||
Column-based actions. Press `V` (shift-v) to enter Select mode, then use `h`/`l` or arrow keys to move between column headers. The active column is visually highlighted. Press `Enter` or `Space` to trigger that column's current action.
|
||||
|
||||
| Column | Filter action |
|
||||
|-------------------------------|---------------------------------------------------------------------------|
|
||||
| Inst | Cycle availability filter |
|
||||
| Model | Enter search mode |
|
||||
| Provider | Open provider popup |
|
||||
| Params | Open parameter-size bucket popup (<3B, 3-7B, 7-14B, 14-30B, 30-70B, 70B+) |
|
||||
| Score, tok/s, Mem%, Ctx, Date | Sort by that column |
|
||||
| Quant | Open quantization popup |
|
||||
| Mode | Open run-mode popup (GPU, MoE, CPU+GPU, CPU) |
|
||||
| Fit | Cycle fit filter |
|
||||
| Use Case | Open use-case popup |
|
||||
|
||||
Row navigation still works in Select mode so you can see the effect of actions as you apply them: `j`/`k`, arrow keys, `Ctrl-U`, `Ctrl-D`, `PageUp`, `PageDown`, `Home`, and `End`. Press `Esc` to return to Normal mode.
|
||||
|
||||
### TUI Plan mode (`p`)
|
||||
|
||||
Plan mode inverts normal fit analysis: instead of asking "what fits my hardware?", it estimates "what hardware is needed for this model config?".
|
||||
|
||||
Use `p` on a selected row, then:
|
||||
|
||||
| Key | Action |
|
||||
|------------------------|-----------------------------------------------------------|
|
||||
| `Tab` / `j` / `k` | Move between editable fields (Context, Quant, Target TPS) |
|
||||
| `Left` / `Right` | Move cursor in current field |
|
||||
| Type | Edit current field |
|
||||
| `Backspace` / `Delete` | Remove characters |
|
||||
| `Ctrl-U` | Clear current field |
|
||||
| `Esc` or `q` | Exit Plan mode |
|
||||
|
||||
Plan mode shows estimates for:
|
||||
- minimum and recommended VRAM/RAM/CPU cores
|
||||
- feasible run paths (GPU, CPU offload, CPU-only)
|
||||
- upgrade deltas to reach better fit targets
|
||||
|
||||
### Hardware Simulation (`S`)
|
||||
|
||||
Press `S` to open the hardware simulation popup. Override RAM, VRAM, and CPU core count to see which models would fit on different target hardware. All model scores, fit levels, and speed estimates are recalculated instantly against the simulated specs.
|
||||
|
||||
|
||||
|
||||
| Key | Action |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `Tab` / `j` / `k` | Switch between RAM, VRAM, CPU fields |
|
||||
| Type digits | Edit the selected field |
|
||||
| `Enter` | Apply simulation |
|
||||
| `Ctrl-R` | Reset to real detected hardware |
|
||||
| `Esc` | Cancel and close |
|
||||
|
||||
When simulation is active, a `SIM` badge appears in the system bar and status bar. The entire model table reflects the simulated hardware until you reset.
|
||||
|
||||
### Advanced Configuration (`A`)
|
||||
|
||||
Press `A` to open the Advanced Configuration popup. This panel lets you tune the parameters behind TPS estimation, run mode penalties, and composite scoring — addressing [issue #449](https://github.com/AlexsJones/llmfit/issues/449) where tok/s was overestimated for certain models (e.g., Qwen3 30B).
|
||||
|
||||
All changes are applied immediately and the model table is recalculated. Close with `Esc` to accept or `Ctrl-R` to reset to defaults.
|
||||
|
||||
| Field | Description | Default |
|
||||
|--------------------|-------------------------------------------------------------------------|---------|
|
||||
| **Efficiency** | Global efficiency factor for bandwidth-based TPS. Accounts for overhead | `0.55` |
|
||||
| **GPU factor** | Speed multiplier for pure GPU inference | `1.0` |
|
||||
| **CPU Offload** | Speed multiplier when weights spill to system RAM | `0.5` |
|
||||
| **MoE Offload** | Speed multiplier for Mixture-of-Experts expert switching | `0.8` |
|
||||
| **Tensor Par** | Speed multiplier for tensor-parallel inference | `0.9` |
|
||||
| **CPU Only** | Speed multiplier for CPU-only execution | `0.3` |
|
||||
| **Context cap** | Max context length used for memory estimation (leave blank for default) | `auto` |
|
||||
|
||||
| Key | Action |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `Tab` / `j` / `k` | Switch between fields |
|
||||
| Type digits / `.` | Edit the selected field |
|
||||
| `Left` / `Right` | Move cursor within the field |
|
||||
| `Backspace` / `Delete` | Remove characters |
|
||||
| `Ctrl-U` | Clear the current field |
|
||||
| `Enter` | Apply changes and recalculate all scores|
|
||||
| `Esc` / `q` | Close without applying |
|
||||
|
||||
### Download Manager (`D`)
|
||||
|
||||
Press `D` to open the Download Manager view. This full-screen view replaces the main model table and provides three sections:
|
||||
|
||||
- **Active Download** — shows the current download in progress with a progress bar, model name, and status message.
|
||||
- **Config** — displays (and allows editing) the GGUF models directory. The configured path persists across sessions.
|
||||
- **History** — a navigable list of past downloads (newest first) with model name, provider, status, and date. Failed downloads can be removed from history, and successful downloads can be deleted from the provider.
|
||||
|
||||
Use `Tab` / `Shift-Tab` to cycle focus between sections.
|
||||
|
||||
| Key | Action |
|
||||
|------------------------|--------------------------------------------------|
|
||||
| `Tab` / `Shift-Tab` | Cycle focus: Active → Config → History |
|
||||
| `j` / `k` or arrows | Navigate the history list (when History focused) |
|
||||
| `x` | Delete selected model (prompts for confirmation) |
|
||||
| `y` / `n` | Confirm or cancel deletion |
|
||||
| `e` | Edit download directory (when Config focused) |
|
||||
| `Enter` | Confirm directory edit |
|
||||
| `Esc` / `D` / `q` | Close and return to the model table |
|
||||
|
||||
For failed downloads (e.g. 404 errors), `x` removes the entry from history. For successful downloads, it deletes the model from the provider (supported for Ollama and llama.cpp).
|
||||
|
||||
### Community Leaderboard (`b`)
|
||||
|
||||
Press `b` to open the Community Leaderboard view. Instead of relying solely on llmfit's theoretical speed estimates, this view shows **real-world performance data** from other users with the same hardware — actual measured tok/s, time-to-first-token, and peak VRAM usage.
|
||||
|
||||
|
||||
|
||||
Data is sourced from [localmaxxing.com](https://localmaxxing.com), a community benchmark database. When you open the view, llmfit auto-detects your hardware (GPU model, VRAM tier, Apple Silicon chip family, OS) and queries for matching results.
|
||||
|
||||
| Column | Description |
|
||||
|--------------|----------------------------------------------------------|
|
||||
| **Model** | HuggingFace model ID |
|
||||
| **Engine** | Inference runtime used (llama.cpp, vLLM, Ollama, MLX...) |
|
||||
| **Quant** | Quantization format (Q4_K_M, Q8_0, etc.) |
|
||||
| **tok/s** | Measured output token generation speed |
|
||||
| **Total t/s**| Total throughput (prompt + generation) |
|
||||
| **TTFT** | Time to first token (latency) |
|
||||
| **VRAM** | Peak memory usage during inference |
|
||||
| **Ctx** | Context length used in the benchmark |
|
||||
| **User** | Submitter (verified users marked with `*`) |
|
||||
|
||||
| Key | Action |
|
||||
|------------------------|-----------------------------------------|
|
||||
| `j` / `k` or arrows | Navigate results |
|
||||
| `H` | Open hardware picker (browse any GPU) |
|
||||
| `r` | Refresh / re-fetch from API |
|
||||
| `b` / `q` / `Esc` | Close and return to model table |
|
||||
|
||||
Press `H` to open the hardware picker — a scrollable list of 27 popular GPUs and chips (RTX 5090 through CPU-only, plus Apple Silicon M1–M4 variants, AMD RX/MI series, and NVIDIA datacenter cards). Select one to instantly load benchmarks for that hardware, even if it's not what you're running on. Select "My Hardware (auto-detect)" to go back to your own system.
|
||||
|
||||
#### API key setup
|
||||
|
||||
Public benchmarks work without authentication. For full access, provide your [localmaxxing.com](https://localmaxxing.com) API key:
|
||||
|
||||
```sh
|
||||
# Via environment variable (recommended)
|
||||
export LOCALMAXXING_API_KEY="bhk_your_key_here"
|
||||
llmfit
|
||||
|
||||
# Or via CLI flag
|
||||
llmfit --api-key "bhk_your_key_here"
|
||||
```
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `LOCALMAXXING_API_KEY` | Bearer token for localmaxxing.com API |
|
||||
|
||||
### Inference Bench (`I`)
|
||||
|
||||
Press `I` (uppercase) to open the Inference Bench view. This runs **live inference benchmarks against your locally running providers** — Ollama, vLLM, and MLX — measuring time-to-first-token (TTFT), tokens per second (TPS), and total latency with real inference requests.
|
||||
|
||||
Unlike the Community Leaderboard (which shows crowd-sourced data from other users), Inference Bench measures your actual hardware with your actual models.
|
||||
|
||||
#### TUI usage
|
||||
|
||||
| Key | Action |
|
||||
|-----|--------|
|
||||
| `I` | Open inference bench (auto-detects provider and runs benchmarks) |
|
||||
| `I` (again) | Rerun benchmarks from within the bench view |
|
||||
| `j` / `k` or arrows | Navigate model results |
|
||||
| `Enter` | Open detail view for selected model |
|
||||
| `r` | Switch to routing matrix view |
|
||||
| `q` / `Esc` | Close bench view |
|
||||
|
||||
Results are cached to `~/.config/llmfit/bench-cache.json` and loaded instantly on subsequent opens.
|
||||
|
||||
#### CLI usage
|
||||
|
||||
```sh
|
||||
# Auto-detect provider and benchmark
|
||||
llmfit bench
|
||||
|
||||
# Benchmark all discovered models across all running providers
|
||||
llmfit bench --all
|
||||
|
||||
# Benchmark a specific model via Ollama
|
||||
llmfit bench --provider ollama llama3.2
|
||||
|
||||
# Override endpoint URL
|
||||
llmfit bench --provider ollama --url http://my-server:11434 llama3.2
|
||||
|
||||
# Override vLLM endpoint
|
||||
llmfit bench --provider vllm --url http://localhost:8000
|
||||
|
||||
# Output as JSON (for scripting)
|
||||
llmfit bench --json
|
||||
|
||||
# Run quality benchmarks (role-based scoring for routing)
|
||||
llmfit bench --quality
|
||||
|
||||
# Output routing matrix
|
||||
llmfit bench --quality --routing
|
||||
```
|
||||
|
||||
#### Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `OLLAMA_HOST` | `http://localhost:11434` | Ollama API base URL |
|
||||
| `VLLM_PORT` | `8000` | vLLM server port (used as `http://localhost:$VLLM_PORT`) |
|
||||
|
||||
### Themes
|
||||
|
||||
Press `t` to cycle through 10 built-in color themes. Your selection is saved automatically to `~/.config/llmfit/theme` and restored on next launch.
|
||||
|
||||
| Theme | Description |
|
||||
|--------------------------|---------------------------------------------------|
|
||||
| **Default** | Original llmfit colors |
|
||||
| **Dracula** | Dark purple background with pastel accents |
|
||||
| **Solarized** | Ethan Schoonover's Solarized Dark palette |
|
||||
| **Nord** | Arctic, cool blue-gray tones |
|
||||
| **Monokai** | Monokai Pro warm syntax colors |
|
||||
| **Gruvbox** | Retro groove palette with warm earth tones |
|
||||
| **Catppuccin Latte** | 🌻 Light theme — harmonious pastel inversion |
|
||||
| **Catppuccin Frappé** | 🪴 Low-contrast dark — muted, subdued aesthetic |
|
||||
| **Catppuccin Macchiato** | 🌺 Medium-contrast dark — gentle, soothing tones |
|
||||
| **Catppuccin Mocha** | 🌿 Darkest variant — cozy with color-rich accents |
|
||||
|
||||
### Web dashboard
|
||||
|
||||
When you run `llmfit` in non-JSON mode, it automatically starts a background web dashboard on `0.0.0.0:8787`. Open it in any browser on the same network:
|
||||
|
||||
```
|
||||
http://<your-machine-ip>:8787
|
||||
```
|
||||
|
||||
Override the host or port with environment variables:
|
||||
|
||||
```sh
|
||||
LLMFIT_DASHBOARD_HOST=0.0.0.0 LLMFIT_DASHBOARD_PORT=9000 llmfit
|
||||
```
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `LLMFIT_DASHBOARD_HOST` | `0.0.0.0` | Interface to bind the dashboard server |
|
||||
| `LLMFIT_DASHBOARD_PORT` | `8787` | Port to bind the dashboard server |
|
||||
|
||||
To disable the auto-started dashboard, pass `--no-dashboard`:
|
||||
|
||||
```sh
|
||||
llmfit --no-dashboard
|
||||
```
|
||||
Generated
+61
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1772624091,
|
||||
"narHash": "sha256-QKyJ0QGWBn6r0invrMAK8dmJoBYWoOWy7lN+UHzW1jc=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "80bdc1e5ce51f56b19791b52b2901187931f5353",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
description = "Hundreds of models & providers. One command to find what runs on your hardware.";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils }:
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
version = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).workspace.package.version;
|
||||
in
|
||||
{
|
||||
packages.default = pkgs.rustPlatform.buildRustPackage {
|
||||
pname = "llmfit";
|
||||
inherit version;
|
||||
|
||||
src = ./.;
|
||||
|
||||
cargoLock.lockFile = ./Cargo.lock;
|
||||
|
||||
# Build only the TUI binary (default workspace members exclude desktop)
|
||||
cargoBuildFlags = [ "--package" "llmfit" ];
|
||||
cargoTestFlags = [ "--package" "llmfit" "--package" "llmfit-core" ];
|
||||
|
||||
meta = with pkgs.lib; {
|
||||
description = "Matches LLM models to your hardware capabilities";
|
||||
homepage = "https://github.com/AlexsJones/llmfit";
|
||||
license = licenses.mit;
|
||||
maintainers = [ ];
|
||||
mainProgram = "llmfit";
|
||||
};
|
||||
};
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
buildInputs = with pkgs; [
|
||||
rustc
|
||||
cargo
|
||||
rust-analyzer
|
||||
clippy
|
||||
rustfmt
|
||||
];
|
||||
};
|
||||
});
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>llmfit</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>llmfit</h1>
|
||||
<p>Match LLM models to your hardware.</p>
|
||||
<pre>curl -fsSL https://llmfit.axjns.dev/install.sh | sh</pre>
|
||||
<p><a href="https://github.com/AlexsJones/llmfit">GitHub</a></p>
|
||||
</body>
|
||||
</html>
|
||||
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/bin/sh
|
||||
# llmfit installer
|
||||
# Usage: curl -fsSL https://raw.githubusercontent.com/AlexsJones/llmfit/main/install.sh | sh
|
||||
# curl -fsSL ... | sh -s -- --local # Install to ~/.local/bin (no sudo)
|
||||
#
|
||||
# Downloads the latest llmfit release from GitHub and installs
|
||||
# the binary to /usr/local/bin (or ~/.local/bin with --local or if no sudo).
|
||||
# Supports piped execution: sudo prompts read from /dev/tty when stdin is a pipe.
|
||||
|
||||
set -e
|
||||
|
||||
REPO="AlexsJones/llmfit"
|
||||
BINARY="llmfit"
|
||||
LOCAL_INSTALL=""
|
||||
|
||||
# --- helpers ---
|
||||
|
||||
info() { printf ' \033[1;34m>\033[0m %s\n' "$*"; }
|
||||
warn() { printf ' \033[1;33m>\033[0m %s\n' "$*"; }
|
||||
err() { printf ' \033[1;31m!\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || err "Required tool '$1' not found. Please install it and try again."
|
||||
}
|
||||
|
||||
# --- parse arguments ---
|
||||
|
||||
parse_args() {
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--local|-l)
|
||||
LOCAL_INSTALL="1"
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: install.sh [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --local, -l Install to ~/.local/bin (no sudo required)"
|
||||
echo " --help, -h Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
warn "Unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
}
|
||||
|
||||
# --- detect platform ---
|
||||
|
||||
detect_platform() {
|
||||
OS="$(uname -s)"
|
||||
ARCH="$(uname -m)"
|
||||
|
||||
case "$OS" in
|
||||
Linux) OS="unknown-linux-musl" ;;
|
||||
Darwin) OS="apple-darwin" ;;
|
||||
*) err "Unsupported OS: $OS" ;;
|
||||
esac
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64|amd64) ARCH="x86_64" ;;
|
||||
aarch64|arm64) ARCH="aarch64" ;;
|
||||
*) err "Unsupported architecture: $ARCH" ;;
|
||||
esac
|
||||
|
||||
PLATFORM="${ARCH}-${OS}"
|
||||
}
|
||||
|
||||
# --- fetch latest release ---
|
||||
|
||||
fetch_latest_tag() {
|
||||
need curl
|
||||
need tar
|
||||
|
||||
# Use the releases redirect instead of the API to avoid GitHub's
|
||||
# 60-request/hour rate limit on unauthenticated API calls (403).
|
||||
TAG="$(curl -fsSI "https://github.com/${REPO}/releases/latest" 2>/dev/null \
|
||||
| grep -i '^location:' \
|
||||
| head -1 \
|
||||
| sed 's|.*/tag/||' \
|
||||
| tr -d '\r\n')"
|
||||
|
||||
[ -n "$TAG" ] || err "Could not determine latest release. Check https://github.com/${REPO}/releases"
|
||||
}
|
||||
|
||||
# --- checksum verification ---
|
||||
|
||||
verify_checksum() {
|
||||
CHECKSUM_FILE="${TMPDIR}/${ASSET}.sha256"
|
||||
|
||||
# Attempt to download the checksum file (-f exits non-zero on HTTP 4xx/5xx)
|
||||
if ! curl -fsSL --max-time 10 "${URL}.sha256" -o "$CHECKSUM_FILE" 2>/dev/null; then
|
||||
warn "No checksum file found for this release — skipping integrity check"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Verifying checksum..."
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
(cd "$TMPDIR" && sha256sum -c "${ASSET}.sha256" --quiet) \
|
||||
|| err "Checksum verification failed. The download may be corrupted or tampered with."
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
(cd "$TMPDIR" && shasum -a 256 -q -c "${ASSET}.sha256") \
|
||||
|| err "Checksum verification failed. The download may be corrupted or tampered with."
|
||||
else
|
||||
warn "Neither sha256sum nor shasum available — skipping integrity check"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- download and install ---
|
||||
|
||||
install() {
|
||||
ASSET="${BINARY}-${TAG}-${PLATFORM}.tar.gz"
|
||||
URL="https://github.com/${REPO}/releases/download/${TAG}/${ASSET}"
|
||||
|
||||
TMPDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
info "Downloading ${BINARY} ${TAG} for ${PLATFORM}..."
|
||||
curl -fsSL "$URL" -o "${TMPDIR}/${ASSET}" \
|
||||
|| err "Download failed. Asset '${ASSET}' may not exist for your platform.\n Check: https://github.com/${REPO}/releases/tag/${TAG}"
|
||||
|
||||
verify_checksum
|
||||
|
||||
info "Extracting..."
|
||||
tar -xzf "${TMPDIR}/${ASSET}" -C "$TMPDIR"
|
||||
|
||||
# Find the binary in the extracted contents
|
||||
BIN="$(find "$TMPDIR" -name "$BINARY" -type f | head -1)"
|
||||
[ -n "$BIN" ] || err "Binary not found in archive. Release asset may have an unexpected layout."
|
||||
chmod +x "$BIN"
|
||||
|
||||
# Determine install directory
|
||||
if [ -n "$LOCAL_INSTALL" ]; then
|
||||
# User explicitly requested local install
|
||||
INSTALL_DIR="${HOME}/.local/bin"
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
info "Installing to ${INSTALL_DIR} (--local mode)..."
|
||||
elif [ -w /usr/local/bin ]; then
|
||||
# /usr/local/bin is writable without sudo
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
elif command -v sudo >/dev/null 2>&1; then
|
||||
# sudo is available — use /dev/tty for password prompt when stdin is a pipe
|
||||
info "Installing to /usr/local/bin (requires sudo)..."
|
||||
if [ -t 0 ]; then
|
||||
SUDO_ASKPASS="" sudo mv "$BIN" "/usr/local/bin/${BINARY}"
|
||||
elif [ -e /dev/tty ]; then
|
||||
SUDO_ASKPASS="" sudo mv "$BIN" "/usr/local/bin/${BINARY}" </dev/tty
|
||||
else
|
||||
false
|
||||
fi
|
||||
if [ $? -eq 0 ]; then
|
||||
info "Installed ${BINARY} to /usr/local/bin/${BINARY}"
|
||||
return
|
||||
else
|
||||
warn "sudo failed, falling back to ~/.local/bin"
|
||||
INSTALL_DIR="${HOME}/.local/bin"
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
fi
|
||||
else
|
||||
# No write access and no interactive sudo, use local install
|
||||
INSTALL_DIR="${HOME}/.local/bin"
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
info "Installing to ${INSTALL_DIR} (no sudo available)..."
|
||||
fi
|
||||
|
||||
mv "$BIN" "${INSTALL_DIR}/${BINARY}"
|
||||
info "Installed ${BINARY} to ${INSTALL_DIR}/${BINARY}"
|
||||
|
||||
# Check if install dir is in PATH
|
||||
case ":$PATH:" in
|
||||
*":${INSTALL_DIR}:"*) ;;
|
||||
*)
|
||||
warn "Add ${INSTALL_DIR} to your PATH to use '${BINARY}' directly:"
|
||||
echo ""
|
||||
echo " export PATH=\"\$HOME/.local/bin:\$PATH\""
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# --- main ---
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
info "llmfit installer"
|
||||
detect_platform
|
||||
fetch_latest_tag
|
||||
install
|
||||
info "Done. Run '${BINARY}' to get started."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "llmfit-core"
|
||||
version.workspace = true
|
||||
edition = "2024"
|
||||
authors = ["Alex Jones <alex@example.com>"]
|
||||
description = "Core library for llmfit — hardware detection, model fitting, and provider integration"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/AlexsJones/llmfit"
|
||||
homepage = "https://github.com/AlexsJones/llmfit"
|
||||
readme = "../README.md"
|
||||
keywords = ["llm", "hardware", "inference", "models", "gpu"]
|
||||
categories = ["hardware-support"]
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.22"
|
||||
dirs = "6.0"
|
||||
http = "1"
|
||||
regex = "1"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
serde_yml = "0.0"
|
||||
sysinfo = "0.39"
|
||||
ureq = { version = "3.2", features = ["json"] }
|
||||
which = "8.0.2"
|
||||
|
||||
[build-dependencies]
|
||||
serde_json = "1.0"
|
||||
|
||||
[dev-dependencies]
|
||||
jsonschema = { version = "0.46", default-features = false }
|
||||
@@ -0,0 +1,58 @@
|
||||
//! Aggregate community benchmark submissions (data/community/<slug>/*.json)
|
||||
//! into a single JSON array embedded in the binary. This is what closes the
|
||||
//! contribution loop: a submission merged into the repo ships to every user
|
||||
//! in the next release, with no CI step or network fetch involved.
|
||||
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=data/community");
|
||||
|
||||
let community_dir = Path::new("data/community");
|
||||
let mut files: Vec<PathBuf> = Vec::new();
|
||||
if let Ok(entries) = fs::read_dir(community_dir) {
|
||||
for entry in entries.flatten() {
|
||||
let slug_dir = entry.path();
|
||||
if !slug_dir.is_dir() {
|
||||
continue; // README.md, schema.json
|
||||
}
|
||||
if let Ok(subs) = fs::read_dir(&slug_dir) {
|
||||
for sub in subs.flatten() {
|
||||
let p = sub.path();
|
||||
if p.extension().and_then(|e| e.to_str()) == Some("json") {
|
||||
files.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Deterministic embed order regardless of directory iteration order.
|
||||
files.sort();
|
||||
|
||||
let mut payloads: Vec<serde_json::Value> = Vec::new();
|
||||
for f in &files {
|
||||
let Ok(text) = fs::read_to_string(f) else {
|
||||
println!(
|
||||
"cargo:warning=community submission unreadable, skipped: {}",
|
||||
f.display()
|
||||
);
|
||||
continue;
|
||||
};
|
||||
match serde_json::from_str::<serde_json::Value>(&text) {
|
||||
Ok(v) => payloads.push(v),
|
||||
// CI validates submissions on PR; a bad file here should never
|
||||
// happen, but a warning beats breaking every build.
|
||||
Err(e) => println!(
|
||||
"cargo:warning=community submission invalid JSON, skipped: {}: {e}",
|
||||
f.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
let out = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR set by cargo"))
|
||||
.join("community_benchmarks.json");
|
||||
let json = serde_json::to_string(&payloads).expect("serialize community aggregate");
|
||||
fs::write(&out, json).expect("write community aggregate");
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"_comment": "Frontier model baseline scores. Run `llmfit bench --all --quality --provider <x>` to regenerate.",
|
||||
"_updated": "2026-03-22",
|
||||
"_note": "Scores are from running the same benchmarks.yaml tests. Update periodically as models change.",
|
||||
"baselines": [
|
||||
{
|
||||
"model": "claude-sonnet-4-5",
|
||||
"provider": "anthropic",
|
||||
"roles": {
|
||||
"general": { "quality": 9.2, "speed": 85.0, "composite": 9.5 },
|
||||
"fast": { "quality": 8.5, "speed": 95.0, "composite": 9.3 },
|
||||
"coding": { "quality": 9.5, "speed": 80.0, "composite": 9.6 },
|
||||
"ui-coding": { "quality": 9.0, "speed": 78.0, "composite": 9.2 },
|
||||
"security-audit": { "quality": 9.3, "speed": 75.0, "composite": 9.3 },
|
||||
"reasoning": { "quality": 9.5, "speed": 70.0, "composite": 9.4 },
|
||||
"critique": { "quality": 9.2, "speed": 72.0, "composite": 9.2 },
|
||||
"creative": { "quality": 8.8, "speed": 80.0, "composite": 9.0 },
|
||||
"writing": { "quality": 9.0, "speed": 82.0, "composite": 9.2 },
|
||||
"research": { "quality": 9.3, "speed": 75.0, "composite": 9.3 },
|
||||
"vision": { "quality": 8.5, "speed": 70.0, "composite": 8.8 },
|
||||
"image-gen": { "quality": 8.0, "speed": 78.0, "composite": 8.5 },
|
||||
"critical-ops": { "quality": 9.2, "speed": 73.0, "composite": 9.2 },
|
||||
"tool-calling": { "quality": 9.5, "speed": 85.0, "composite": 9.6 },
|
||||
"structured-output": { "quality": 9.8, "speed": 82.0, "composite": 9.8 },
|
||||
"code-editing": { "quality": 9.3, "speed": 78.0, "composite": 9.3 },
|
||||
"error-recovery": { "quality": 9.0, "speed": 76.0, "composite": 9.1 },
|
||||
"planning": { "quality": 9.2, "speed": 74.0, "composite": 9.2 },
|
||||
"long-context": { "quality": 9.5, "speed": 80.0, "composite": 9.5 }
|
||||
},
|
||||
"overall": { "quality": 9.2, "speed": 78.0, "composite": 9.3 }
|
||||
},
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"provider": "openai",
|
||||
"roles": {
|
||||
"general": { "quality": 8.8, "speed": 90.0, "composite": 9.2 },
|
||||
"fast": { "quality": 8.0, "speed": 100.0, "composite": 9.1 },
|
||||
"coding": { "quality": 9.0, "speed": 85.0, "composite": 9.3 },
|
||||
"ui-coding": { "quality": 8.8, "speed": 82.0, "composite": 9.0 },
|
||||
"security-audit": { "quality": 8.5, "speed": 80.0, "composite": 8.8 },
|
||||
"reasoning": { "quality": 9.0, "speed": 75.0, "composite": 9.0 },
|
||||
"critique": { "quality": 8.5, "speed": 78.0, "composite": 8.7 },
|
||||
"creative": { "quality": 8.5, "speed": 85.0, "composite": 8.8 },
|
||||
"writing": { "quality": 8.8, "speed": 88.0, "composite": 9.0 },
|
||||
"research": { "quality": 8.8, "speed": 80.0, "composite": 8.9 },
|
||||
"vision": { "quality": 9.0, "speed": 75.0, "composite": 9.0 },
|
||||
"image-gen": { "quality": 8.5, "speed": 82.0, "composite": 8.8 },
|
||||
"critical-ops": { "quality": 8.8, "speed": 78.0, "composite": 8.9 },
|
||||
"tool-calling": { "quality": 9.5, "speed": 90.0, "composite": 9.6 },
|
||||
"structured-output": { "quality": 9.5, "speed": 88.0, "composite": 9.5 },
|
||||
"code-editing": { "quality": 8.8, "speed": 82.0, "composite": 8.9 },
|
||||
"error-recovery": { "quality": 8.5, "speed": 80.0, "composite": 8.7 },
|
||||
"planning": { "quality": 8.8, "speed": 78.0, "composite": 8.9 },
|
||||
"long-context": { "quality": 9.0, "speed": 85.0, "composite": 9.1 }
|
||||
},
|
||||
"overall": { "quality": 8.8, "speed": 83.0, "composite": 9.0 }
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,99 @@
|
||||
# Community benchmarks
|
||||
|
||||
This directory collects benchmark results contributed by llmfit users via:
|
||||
|
||||
```sh
|
||||
llmfit bench --all --share
|
||||
```
|
||||
|
||||
`--share` runs a benchmark sweep, shows you the exact JSON payloads, asks for
|
||||
confirmation, then opens a pull request adding the files here — **without
|
||||
needing the `gh` CLI**. Authentication uses the GitHub OAuth *device flow* (the
|
||||
same mechanism `gh auth login` uses); a `GITHUB_TOKEN` / `GH_TOKEN` env var is
|
||||
used automatically when present (e.g. in CI).
|
||||
|
||||
Bench runs made **without** `--share` are kept in a local store on the user's
|
||||
machine; a later `llmfit bench --share` (with nothing else to bench) uploads
|
||||
that stored backlog in one PR, so declining to share never discards data.
|
||||
|
||||
Preview what would be submitted without contacting GitHub:
|
||||
|
||||
```sh
|
||||
llmfit bench --all --share --dry-run
|
||||
```
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
community/
|
||||
<hardware-slug>/
|
||||
<unix-timestamp>-<hash>.json
|
||||
```
|
||||
|
||||
Files are namespaced by hardware and carry a content hash so concurrent
|
||||
submissions never collide. Each file name mirrors the contributor's local
|
||||
store entry, which makes submissions idempotent: if a contributor already has
|
||||
an open benchmark PR, new results are appended to it (instead of opening
|
||||
another PR), and a retry after a partial failure skips files that already
|
||||
landed rather than duplicating them.
|
||||
|
||||
## What happens after merge
|
||||
|
||||
Submissions are aggregated by `llmfit-core/build.rs` and **embedded into the
|
||||
binary**, so every merged submission ships in the next release. Users on
|
||||
identical hardware (same CPU + GPU) then get:
|
||||
|
||||
- your runs on their **benchmark page**, attributed `llmfit community`
|
||||
- **measured ✓ tok/s** in the fit table for the models you benched
|
||||
(below their own local runs, above localmaxxing medians and estimates)
|
||||
- **calibrated estimates** for every other model, anchored on your runs —
|
||||
a fresh install benefits before its user ever benchmarks anything
|
||||
|
||||
## Validation
|
||||
|
||||
Every PR touching this directory runs the **Community Benchmarks** workflow
|
||||
(`scripts/validate_community_benchmarks.py`): schema conformance against
|
||||
[`schema.json`](./schema.json), path/naming conventions, and cross-field
|
||||
sanity checks (tps ordering, plausible hardware bounds, submission
|
||||
timestamps). Hand-crafted submissions are welcome as long as they pass the
|
||||
same checks the generated ones do.
|
||||
|
||||
## Format
|
||||
|
||||
Each file conforms to [`schema.json`](./schema.json). Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"submittedAtUnix": 1752127200,
|
||||
"tool": { "name": "llmfit", "version": "1.0.0" },
|
||||
"hardware": {
|
||||
"hwClass": "DISCRETE_GPU",
|
||||
"hardwareName": "NVIDIA GeForce RTX 4090",
|
||||
"memTierGb": 24,
|
||||
"vramGb": 24.0,
|
||||
"gpuCount": 1,
|
||||
"unifiedMemory": false,
|
||||
"cpu": "AMD Ryzen 9 7950X",
|
||||
"cpuCores": 32,
|
||||
"ramGb": 64.0,
|
||||
"os": "linux"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"model": "llama3.1:8b",
|
||||
"provider": "ollama",
|
||||
"numRuns": 3,
|
||||
"avgTps": 128.4,
|
||||
"minTps": 121.0,
|
||||
"maxTps": 133.7,
|
||||
"avgTtftMs": 41.2,
|
||||
"avgTotalMs": 812.5,
|
||||
"avgOutputTokens": 104.0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Submissions are validated against the schema and sanity-checked (measurements
|
||||
within physical limits for the reported hardware) before merge.
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"submittedAtUnix": 1783676393,
|
||||
"tool": {
|
||||
"name": "llmfit",
|
||||
"version": "1.0.1"
|
||||
},
|
||||
"hardware": {
|
||||
"hwClass": "UNIFIED",
|
||||
"hardwareName": "Intel Arc Graphics 130V/140V (integrated)",
|
||||
"memTierGb": 32,
|
||||
"vramGb": 30.82,
|
||||
"gpuCount": 1,
|
||||
"unifiedMemory": true,
|
||||
"cpu": "Intel(R) Core(TM) Ultra 7 258V",
|
||||
"cpuCores": 8,
|
||||
"ramGb": 30.82,
|
||||
"os": "linux"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"model": "/home/axjns/.cache/llmfit/models/gemma-3.Q8_0.gguf",
|
||||
"provider": "llamacpp",
|
||||
"numRuns": 3,
|
||||
"avgTps": 3.8,
|
||||
"minTps": 3.68,
|
||||
"maxTps": 3.97,
|
||||
"avgTtftMs": null,
|
||||
"avgTotalMs": 48968.34,
|
||||
"avgOutputTokens": 182.67
|
||||
}
|
||||
]
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"hardware": {
|
||||
"cpu": "Intel(R) Core(TM) Ultra 7 258V",
|
||||
"cpuCores": 8,
|
||||
"gpuCount": 1,
|
||||
"hardwareName": "Intel Arc Graphics 130V/140V (integrated)",
|
||||
"hwClass": "UNIFIED",
|
||||
"memTierGb": 32,
|
||||
"os": "linux",
|
||||
"ramGb": 30.82,
|
||||
"unifiedMemory": true,
|
||||
"vramGb": 30.82
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"avgOutputTokens": 182.67,
|
||||
"avgTotalMs": 48968.34,
|
||||
"avgTps": 3.8,
|
||||
"avgTtftMs": null,
|
||||
"maxTps": 3.97,
|
||||
"minTps": 3.68,
|
||||
"model": "/home/axjns/.cache/llmfit/models/gemma-3.Q8_0.gguf",
|
||||
"numRuns": 3,
|
||||
"provider": "llamacpp"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"submittedAtUnix": 1783676393,
|
||||
"tool": {
|
||||
"name": "llmfit",
|
||||
"version": "1.0.1"
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"hardware": {
|
||||
"cpu": "Intel(R) Core(TM) Ultra 7 258V",
|
||||
"cpuCores": 8,
|
||||
"gpuCount": 1,
|
||||
"hardwareName": "Intel Arc Graphics 130V/140V (integrated)",
|
||||
"hwClass": "UNIFIED",
|
||||
"memTierGb": 32,
|
||||
"os": "linux",
|
||||
"ramGb": 30.82,
|
||||
"unifiedMemory": true,
|
||||
"vramGb": 30.82
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"avgOutputTokens": 221.0,
|
||||
"avgTotalMs": 55148.41,
|
||||
"avgTps": 3.97,
|
||||
"avgTtftMs": null,
|
||||
"maxTps": 4.15,
|
||||
"minTps": 3.86,
|
||||
"model": "/home/axjns/.cache/llmfit/models/gemma-3.Q8_0.gguf",
|
||||
"numRuns": 3,
|
||||
"provider": "llamacpp"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"submittedAtUnix": 1783681738,
|
||||
"tool": {
|
||||
"name": "llmfit",
|
||||
"version": "1.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"$id": "https://llmfit.dev/schemas/community-benchmark.json",
|
||||
"title": "llmfit community benchmark submission",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schemaVersion", "submittedAtUnix", "tool", "hardware", "results"],
|
||||
"properties": {
|
||||
"schemaVersion": { "type": "integer", "const": 1 },
|
||||
"submittedAtUnix": { "type": "integer", "minimum": 0 },
|
||||
"tool": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"version": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"hardware": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["hwClass", "gpuCount", "unifiedMemory", "cpu", "cpuCores", "ramGb", "os"],
|
||||
"properties": {
|
||||
"hwClass": { "type": "string", "enum": ["DISCRETE_GPU", "UNIFIED", "CPU_ONLY"] },
|
||||
"hardwareName": { "type": ["string", "null"] },
|
||||
"memTierGb": { "type": ["integer", "null"], "minimum": 0 },
|
||||
"vramGb": { "type": ["number", "null"], "minimum": 0 },
|
||||
"gpuCount": { "type": "integer", "minimum": 0 },
|
||||
"unifiedMemory": { "type": "boolean" },
|
||||
"cpu": { "type": "string" },
|
||||
"cpuCores": { "type": "integer", "minimum": 1 },
|
||||
"ramGb": { "type": "number", "minimum": 0 },
|
||||
"os": { "type": "string", "enum": ["linux", "macos", "windows"] }
|
||||
}
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["model", "provider", "numRuns", "avgTps", "minTps", "maxTps", "avgTotalMs", "avgOutputTokens"],
|
||||
"properties": {
|
||||
"model": { "type": "string", "minLength": 1 },
|
||||
"provider": { "type": "string", "enum": ["ollama", "vllm", "mlx", "llamacpp"] },
|
||||
"numRuns": { "type": "integer", "minimum": 1 },
|
||||
"avgTps": { "type": "number", "minimum": 0, "maximum": 100000 },
|
||||
"minTps": { "type": "number", "minimum": 0, "maximum": 100000 },
|
||||
"maxTps": { "type": "number", "minimum": 0, "maximum": 100000 },
|
||||
"avgTtftMs": { "type": ["number", "null"], "minimum": 0 },
|
||||
"avgTotalMs": { "type": "number", "minimum": 0 },
|
||||
"avgOutputTokens": { "type": "number", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
{
|
||||
"generated_by": "scrape_docker_models.py",
|
||||
"docker_hub_repo_count": 46,
|
||||
"matched_model_count": 35,
|
||||
"models": [
|
||||
{
|
||||
"hf_name": "HuggingFaceTB/SmolLM2-135M",
|
||||
"docker_tag": "ai/smollm2:135m",
|
||||
"docker_repo": "ai/smollm2",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"135M-Q4_K_M",
|
||||
"360M-Q4_K_M",
|
||||
"135M-Q2_K",
|
||||
"360M-F16",
|
||||
"360M-Q4_0",
|
||||
"135M-Q4_0",
|
||||
"135M-F16"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "HuggingFaceTB/SmolLM2-135M-Instruct",
|
||||
"docker_tag": "ai/smollm2:135m",
|
||||
"docker_repo": "ai/smollm2",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"135M-Q4_K_M",
|
||||
"360M-Q4_K_M",
|
||||
"135M-Q2_K",
|
||||
"360M-F16",
|
||||
"360M-Q4_0",
|
||||
"135M-Q4_0",
|
||||
"135M-F16"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-0.5B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:0.5b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-0.5B",
|
||||
"docker_tag": "ai/qwen2.5:0.5b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Gensyn/Qwen2.5-0.5B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:0.5b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-0.6B",
|
||||
"docker_tag": "ai/qwen3:0.6b",
|
||||
"docker_repo": "ai/qwen3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"4B-F16",
|
||||
"4B-UD-Q8_K_XL",
|
||||
"4B-UD-Q4_K_XL",
|
||||
"14B-Q6_K",
|
||||
"30B-A3B-F16",
|
||||
"30B-A3B-Q4_K_M",
|
||||
"0.6B-Q4_0",
|
||||
"0.6B-F16",
|
||||
"0.6B-Q4_K_M",
|
||||
"8B-Q4_0",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "meta-llama/Llama-3.2-1B",
|
||||
"docker_tag": "ai/llama3.2:1b",
|
||||
"docker_repo": "ai/llama3.2",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"3B-Q4_0",
|
||||
"1B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1B-F16",
|
||||
"1B-Q8_0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:1.5b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-1.5B",
|
||||
"docker_tag": "ai/qwen2.5:1.5b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-1.7B-Base",
|
||||
"docker_tag": "ai/qwen3:1.7b",
|
||||
"docker_repo": "ai/qwen3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"4B-F16",
|
||||
"4B-UD-Q8_K_XL",
|
||||
"4B-UD-Q4_K_XL",
|
||||
"14B-Q6_K",
|
||||
"30B-A3B-F16",
|
||||
"30B-A3B-Q4_K_M",
|
||||
"0.6B-Q4_0",
|
||||
"0.6B-F16",
|
||||
"0.6B-Q4_K_M",
|
||||
"8B-Q4_0",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-3B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:3b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "meta-llama/Llama-3.2-3B",
|
||||
"docker_tag": "ai/llama3.2:3b",
|
||||
"docker_repo": "ai/llama3.2",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"3B-Q4_0",
|
||||
"1B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1B-F16",
|
||||
"1B-Q8_0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "google/gemma-3n-E2B-it",
|
||||
"docker_tag": "ai/gemma3n:e2b",
|
||||
"docker_repo": "ai/gemma3n",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"2B-F16",
|
||||
"2B-Q4_K_M",
|
||||
"4B-F16",
|
||||
"4B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "mistralai/Mistral-7B-Instruct-v0.2",
|
||||
"docker_tag": "ai/mistral:7b",
|
||||
"docker_repo": "ai/mistral",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Featherless-Chat-Models/Mistral-7B-Instruct-v0.2",
|
||||
"docker_tag": "ai/mistral:7b",
|
||||
"docker_repo": "ai/mistral",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "mistralai/Mistral-7B-Instruct-v0.3",
|
||||
"docker_tag": "ai/mistral:7b",
|
||||
"docker_repo": "ai/mistral",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:7b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-7B",
|
||||
"docker_tag": "ai/qwen2.5:7b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "google/gemma-3n-E4B-it",
|
||||
"docker_tag": "ai/gemma3n:e4b",
|
||||
"docker_repo": "ai/gemma3n",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"2B-F16",
|
||||
"2B-Q4_K_M",
|
||||
"4B-F16",
|
||||
"4B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "google/gemma-3-12b-it",
|
||||
"docker_tag": "ai/gemma3:12b",
|
||||
"docker_repo": "ai/gemma3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"4B-Q4_K_M",
|
||||
"4B-F16",
|
||||
"270M-UD-IQ2_XXS",
|
||||
"4B",
|
||||
"270M-UD-Q4_K_XL",
|
||||
"270M-F16",
|
||||
"4B-Q4_0",
|
||||
"1B-F16",
|
||||
"1B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "mistralai/Mistral-Nemo-Instruct-2407",
|
||||
"docker_tag": "ai/mistral-nemo",
|
||||
"docker_repo": "ai/mistral-nemo",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"12B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "microsoft/phi-4",
|
||||
"docker_tag": "ai/phi4",
|
||||
"docker_repo": "ai/phi4",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"14B-Q4_0",
|
||||
"14B-F16",
|
||||
"14B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-14B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:14b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-14B",
|
||||
"docker_tag": "ai/qwen3:14b",
|
||||
"docker_repo": "ai/qwen3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"4B-F16",
|
||||
"4B-UD-Q8_K_XL",
|
||||
"4B-UD-Q4_K_XL",
|
||||
"14B-Q6_K",
|
||||
"30B-A3B-F16",
|
||||
"30B-A3B-Q4_K_M",
|
||||
"0.6B-Q4_0",
|
||||
"0.6B-F16",
|
||||
"0.6B-Q4_K_M",
|
||||
"8B-Q4_0",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3.5-27B",
|
||||
"docker_tag": "ai/qwen3.5",
|
||||
"docker_repo": "ai/qwen3.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"397B",
|
||||
"397B-UD-Q4_K_XL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-32B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:32b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3.5-35B-A3B",
|
||||
"docker_tag": "ai/qwen3.5:35b",
|
||||
"docker_repo": "ai/qwen3.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"397B",
|
||||
"397B-UD-Q4_K_XL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "meta-llama/Llama-3.1-70B-Instruct",
|
||||
"docker_tag": "ai/llama3.1:70b",
|
||||
"docker_repo": "ai/llama3.1",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "meta-llama/Llama-3.3-70B-Instruct",
|
||||
"docker_tag": "ai/llama3.3:70b",
|
||||
"docker_repo": "ai/llama3.3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"70B-Q4_0",
|
||||
"70B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen2.5-72B-Instruct",
|
||||
"docker_tag": "ai/qwen2.5:72b",
|
||||
"docker_repo": "ai/qwen2.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"7B-Q4_0",
|
||||
"3B-F16",
|
||||
"3B-Q4_K_M",
|
||||
"1.5B-F16",
|
||||
"0.5B-F16",
|
||||
"7B-F16",
|
||||
"7B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-Coder-Next",
|
||||
"docker_tag": "ai/qwen3-coder-next",
|
||||
"docker_repo": "ai/qwen3-coder-next",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"80B",
|
||||
"80B-Q8_0",
|
||||
"80B-Q4_K_M",
|
||||
"80B-Q5_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-Coder-Next",
|
||||
"docker_tag": "ai/qwen3-coder-next",
|
||||
"docker_repo": "ai/qwen3-coder-next",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"80B",
|
||||
"80B-Q8_0",
|
||||
"80B-Q4_K_M",
|
||||
"80B-Q5_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3.5-122B-A10B",
|
||||
"docker_tag": "ai/qwen3.5:122b",
|
||||
"docker_repo": "ai/qwen3.5",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"397B",
|
||||
"397B-UD-Q4_K_XL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "Qwen/Qwen3-235B-A22B",
|
||||
"docker_tag": "ai/qwen3:235b",
|
||||
"docker_repo": "ai/qwen3",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"4B-F16",
|
||||
"4B-UD-Q8_K_XL",
|
||||
"4B-UD-Q4_K_XL",
|
||||
"14B-Q6_K",
|
||||
"30B-A3B-F16",
|
||||
"30B-A3B-Q4_K_M",
|
||||
"0.6B-Q4_0",
|
||||
"0.6B-F16",
|
||||
"0.6B-Q4_K_M",
|
||||
"8B-Q4_0",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"hf_name": "meta-llama/Llama-3.1-405B-Instruct",
|
||||
"docker_tag": "ai/llama3.1:405b",
|
||||
"docker_repo": "ai/llama3.1",
|
||||
"available_tags": [
|
||||
"latest",
|
||||
"8B-F16",
|
||||
"8B-Q4_K_M"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"id": "onnx-community/Phi-3-mini-4k-instruct",
|
||||
"name": "Phi-3-mini-4k-instruct",
|
||||
"parameters": "3.8B",
|
||||
"license": "mit",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 15300000000,
|
||||
"fp16": 7660000000,
|
||||
"q8": 3820000000,
|
||||
"int8": 3820000000,
|
||||
"uint8": 3820000000,
|
||||
"q4": 2390000000,
|
||||
"q4f16": 2440000000,
|
||||
"bnb4": 2280000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/Qwen2.5-0.5B-Instruct",
|
||||
"name": "Qwen2.5-0.5B-Instruct",
|
||||
"parameters": "0.5B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 1980000000,
|
||||
"fp16": 990000000,
|
||||
"q8": 530000000,
|
||||
"int8": 530000000,
|
||||
"uint8": 530000000,
|
||||
"q4": 380000000,
|
||||
"q4f16": 400000000,
|
||||
"bnb4": 360000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/Qwen2.5-1.5B-Instruct",
|
||||
"name": "Qwen2.5-1.5B-Instruct",
|
||||
"parameters": "1.5B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 6170000000,
|
||||
"fp16": 3090000000,
|
||||
"q8": 1620000000,
|
||||
"int8": 1620000000,
|
||||
"uint8": 1620000000,
|
||||
"q4": 990000000,
|
||||
"q4f16": 1020000000,
|
||||
"bnb4": 950000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/SmolLM2-1.7B-Instruct",
|
||||
"name": "SmolLM2-1.7B-Instruct",
|
||||
"parameters": "1.7B",
|
||||
"license": "apache-2.0",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 6840000000,
|
||||
"fp16": 3420000000,
|
||||
"q8": 1800000000,
|
||||
"int8": 1800000000,
|
||||
"uint8": 1800000000,
|
||||
"q4": 1090000000,
|
||||
"q4f16": 1120000000,
|
||||
"bnb4": 1040000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "onnx-community/gemma-2-2b-it-ONNX",
|
||||
"name": "gemma-2-2b-it",
|
||||
"parameters": "2.6B",
|
||||
"license": "gemma",
|
||||
"format": "onnx",
|
||||
"onnx_files": {
|
||||
"fp32": 10450000000,
|
||||
"fp16": 5230000000,
|
||||
"q8": 2750000000,
|
||||
"int8": 2750000000,
|
||||
"uint8": 2750000000,
|
||||
"q4": 1660000000,
|
||||
"q4f16": 1710000000,
|
||||
"bnb4": 1590000000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,324 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://llmfit.dev/schemas/hf_models.schema.json",
|
||||
"title": "llmfit HuggingFace model catalog",
|
||||
"description": "Schema for llmfit-core/data/hf_models.json",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/model"
|
||||
},
|
||||
"$defs": {
|
||||
"nonNegativeNumberOrNull": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"positiveIntegerOrNull": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"capability": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"vision",
|
||||
"tool_use",
|
||||
"audio",
|
||||
"tts"
|
||||
]
|
||||
},
|
||||
"modelFormat": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"gguf",
|
||||
"awq",
|
||||
"gptq",
|
||||
"autoround",
|
||||
"mlx",
|
||||
"safetensors"
|
||||
]
|
||||
},
|
||||
"quantization": {
|
||||
"type": "string"
|
||||
},
|
||||
"parameterCount": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]+(?:\\.[0-9]+)?[KMBT]$"
|
||||
},
|
||||
"dateOrNull": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"format": "date"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ggufSource": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"repo": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"repo",
|
||||
"provider"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"attentionLayout": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"full": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"linear": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"full",
|
||||
"linear"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"model": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"pattern": "^.+/.+$"
|
||||
},
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"parameter_count": {
|
||||
"$ref": "#/$defs/parameterCount"
|
||||
},
|
||||
"parameters_raw": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"min_ram_gb": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"recommended_ram_gb": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"min_vram_gb": {
|
||||
"$ref": "#/$defs/nonNegativeNumberOrNull"
|
||||
},
|
||||
"quantization": {
|
||||
"$ref": "#/$defs/quantization"
|
||||
},
|
||||
"format": {
|
||||
"$ref": "#/$defs/modelFormat"
|
||||
},
|
||||
"context_length": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"use_case": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"capabilities": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/capability"
|
||||
},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"languages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"pipeline_tag": {
|
||||
"type": "string"
|
||||
},
|
||||
"architecture": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"hf_downloads": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"hf_likes": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"release_date": {
|
||||
"$ref": "#/$defs/dateOrNull"
|
||||
},
|
||||
"is_moe": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"num_experts": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"active_experts": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"active_parameters": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"num_hidden_layers": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"num_attention_heads": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"num_key_value_heads": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"head_dim": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"hidden_size": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"vocab_size": {
|
||||
"$ref": "#/$defs/positiveIntegerOrNull"
|
||||
},
|
||||
"moe_intermediate_size": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"shared_expert_intermediate_size": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"gguf_sources": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/ggufSource"
|
||||
}
|
||||
},
|
||||
"license": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"attention_layout": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/$defs/attentionLayout"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"_audio_backends": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"_audio_rtf_cpu": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"_audio_rtf_gpu": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"_audio_vram_gb": {
|
||||
"$ref": "#/$defs/nonNegativeNumberOrNull"
|
||||
},
|
||||
"_discovered": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"provider",
|
||||
"parameter_count",
|
||||
"min_ram_gb",
|
||||
"recommended_ram_gb",
|
||||
"quantization",
|
||||
"context_length",
|
||||
"use_case"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"is_moe": {
|
||||
"const": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"is_moe"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"required": [
|
||||
"num_experts",
|
||||
"active_experts",
|
||||
"active_parameters"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"_comment": "Curated per-family use-case scores (0-100), aggregated from public leaderboards (LiveCodeBench/HumanEval-class for coding, GPQA/MMLU-Pro-class for reasoning, arena-style preference for chat). Values encode RELATIVE family strength per task, not absolute benchmark numbers. Longest matching substring of the lowercased model name wins. Refresh alongside the weekly model-database update; models without an entry fall back to the name heuristic in fit.rs.",
|
||||
"families": [
|
||||
{ "match": ["qwen3.5-coder", "qwen3-coder", "qwen2.5-coder"], "scores": { "coding": 90, "reasoning": 78, "chat": 72 } },
|
||||
{ "match": ["qwen3.5"], "scores": { "coding": 82, "reasoning": 84, "chat": 82 } },
|
||||
{ "match": ["qwen3"], "scores": { "coding": 78, "reasoning": 80, "chat": 78 } },
|
||||
{ "match": ["deepseek-v4"], "scores": { "coding": 90, "reasoning": 92, "chat": 84 } },
|
||||
{ "match": ["deepseek-v3"], "scores": { "coding": 86, "reasoning": 88, "chat": 82 } },
|
||||
{ "match": ["deepseek-r1", "deepseek-r2"], "scores": { "coding": 84, "reasoning": 93, "chat": 76 } },
|
||||
{ "match": ["llama-4", "llama4"], "scores": { "coding": 74, "reasoning": 80, "chat": 84 } },
|
||||
{ "match": ["llama-3.3", "llama3.3"], "scores": { "coding": 70, "reasoning": 76, "chat": 82 } },
|
||||
{ "match": ["gemma-4", "gemma4"], "scores": { "coding": 72, "reasoning": 76, "chat": 84 } },
|
||||
{ "match": ["gemma-3", "gemma3"], "scores": { "coding": 68, "reasoning": 72, "chat": 80 } },
|
||||
{ "match": ["phi-4", "phi4"], "scores": { "coding": 74, "reasoning": 82, "chat": 72 } },
|
||||
{ "match": ["codestral"], "scores": { "coding": 84, "reasoning": 68, "chat": 62 } },
|
||||
{ "match": ["devstral"], "scores": { "coding": 86, "reasoning": 70, "chat": 60 } },
|
||||
{ "match": ["mixtral"], "scores": { "coding": 66, "reasoning": 72, "chat": 74 } },
|
||||
{ "match": ["mistral"], "scores": { "coding": 68, "reasoning": 70, "chat": 76 } },
|
||||
{ "match": ["glm-4", "glm4"], "scores": { "coding": 80, "reasoning": 82, "chat": 78 } },
|
||||
{ "match": ["kimi"], "scores": { "coding": 78, "reasoning": 84, "chat": 80 } },
|
||||
{ "match": ["granite"], "scores": { "coding": 70, "reasoning": 72, "chat": 70 } },
|
||||
{ "match": ["starcoder"], "scores": { "coding": 78, "reasoning": 55, "chat": 50 } },
|
||||
{ "match": ["olmo"], "scores": { "coding": 62, "reasoning": 66, "chat": 70 } }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
use crate::fit::{InferenceRuntime, ModelFit};
|
||||
use crate::hardware::SystemSpecs;
|
||||
use crate::models::ModelDatabase;
|
||||
use crate::providers::{
|
||||
self, DockerModelRunnerProvider, LlamaCppProvider, LmStudioProvider, MlxProvider,
|
||||
ModelProvider, OllamaProvider, RamaLamaProvider, VllmProvider,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// Aggregated installed-model sets from all supported inference providers.
|
||||
///
|
||||
/// A single point of truth used by both the CLI and the TUI to check which
|
||||
/// models are locally installed. Replaces the scattered `HashSet<String>` fields
|
||||
/// that used to live on each caller's struct.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct InstalledIndex {
|
||||
pub ollama: HashSet<String>,
|
||||
pub ollama_count: usize,
|
||||
pub mlx: HashSet<String>,
|
||||
pub llamacpp: HashSet<String>,
|
||||
pub llamacpp_count: usize,
|
||||
pub docker_mr: HashSet<String>,
|
||||
pub docker_mr_count: usize,
|
||||
pub lmstudio: HashSet<String>,
|
||||
pub lmstudio_count: usize,
|
||||
pub vllm: HashSet<String>,
|
||||
pub vllm_count: usize,
|
||||
pub ramalama: HashSet<String>,
|
||||
pub ramalama_count: usize,
|
||||
}
|
||||
|
||||
impl InstalledIndex {
|
||||
/// Build an empty index — used as a placeholder while providers load.
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
ollama: HashSet::new(),
|
||||
ollama_count: 0,
|
||||
mlx: HashSet::new(),
|
||||
llamacpp: HashSet::new(),
|
||||
llamacpp_count: 0,
|
||||
docker_mr: HashSet::new(),
|
||||
docker_mr_count: 0,
|
||||
lmstudio: HashSet::new(),
|
||||
lmstudio_count: 0,
|
||||
vllm: HashSet::new(),
|
||||
vllm_count: 0,
|
||||
ramalama: HashSet::new(),
|
||||
ramalama_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect installed models across all providers in parallel.
|
||||
///
|
||||
/// Each provider query is issued on its own thread so that a single
|
||||
/// offline/slow backend (worst case ~1.5 s timeout) doesn't serialize
|
||||
/// into ~9 s of total blocking time for the CLI path.
|
||||
pub fn detect_all() -> Self {
|
||||
std::thread::scope(|s| {
|
||||
let ollama = s.spawn(|| {
|
||||
let p = OllamaProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
let mlx = s.spawn(|| MlxProvider::new().installed_models());
|
||||
let llamacpp = s.spawn(|| {
|
||||
let p = LlamaCppProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
let docker_mr = s.spawn(|| {
|
||||
let p = DockerModelRunnerProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
let lmstudio = s.spawn(|| {
|
||||
let p = LmStudioProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
let vllm = s.spawn(|| {
|
||||
let p = VllmProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
let ramalama = s.spawn(|| {
|
||||
let p = RamaLamaProvider::new();
|
||||
p.installed_models_counted()
|
||||
});
|
||||
|
||||
let (ollama, ollama_count) = ollama.join().unwrap();
|
||||
let mlx = mlx.join().unwrap();
|
||||
let (llamacpp, llamacpp_count) = llamacpp.join().unwrap();
|
||||
let (docker_mr, docker_mr_count) = docker_mr.join().unwrap();
|
||||
let (lmstudio, lmstudio_count) = lmstudio.join().unwrap();
|
||||
let (vllm, vllm_count) = vllm.join().unwrap();
|
||||
let (ramalama, ramalama_count) = ramalama.join().unwrap();
|
||||
|
||||
Self {
|
||||
ollama,
|
||||
ollama_count,
|
||||
mlx,
|
||||
llamacpp,
|
||||
llamacpp_count,
|
||||
docker_mr,
|
||||
docker_mr_count,
|
||||
lmstudio,
|
||||
lmstudio_count,
|
||||
vllm,
|
||||
vllm_count,
|
||||
ramalama,
|
||||
ramalama_count,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` when the model is installed in **any** provider.
|
||||
pub fn is_installed(&self, model_name: &str) -> bool {
|
||||
providers::is_model_installed(model_name, &self.ollama)
|
||||
|| providers::is_model_installed_mlx(model_name, &self.mlx)
|
||||
|| providers::is_model_installed_llamacpp(model_name, &self.llamacpp)
|
||||
|| providers::is_model_installed_docker_mr(model_name, &self.docker_mr)
|
||||
|| providers::is_model_installed_lmstudio(model_name, &self.lmstudio)
|
||||
|| providers::is_model_installed_vllm(model_name, &self.vllm)
|
||||
|| providers::is_model_installed_ramalama(model_name, &self.ramalama)
|
||||
}
|
||||
|
||||
/// Returns the display names of all providers that have this model
|
||||
/// installed. Used by the detail panel in the TUI.
|
||||
pub fn installed_providers(&self, model_name: &str) -> Vec<&'static str> {
|
||||
let mut out = Vec::new();
|
||||
if providers::is_model_installed(model_name, &self.ollama) {
|
||||
out.push("Ollama");
|
||||
}
|
||||
if providers::is_model_installed_mlx(model_name, &self.mlx) {
|
||||
out.push("MLX");
|
||||
}
|
||||
if providers::is_model_installed_llamacpp(model_name, &self.llamacpp) {
|
||||
out.push("llama.cpp");
|
||||
}
|
||||
if providers::is_model_installed_docker_mr(model_name, &self.docker_mr) {
|
||||
out.push("Docker");
|
||||
}
|
||||
if providers::is_model_installed_lmstudio(model_name, &self.lmstudio) {
|
||||
out.push("LM Studio");
|
||||
}
|
||||
if providers::is_model_installed_vllm(model_name, &self.vllm) {
|
||||
out.push("vLLM");
|
||||
}
|
||||
if providers::is_model_installed_ramalama(model_name, &self.ramalama) {
|
||||
out.push("RamaLama");
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a complete `Vec<ModelFit>` with installed markers populated.
|
||||
///
|
||||
/// Filters models that are backend-incompatible, runs fit analysis, marks
|
||||
/// each fit's `installed` flag from the given index, and returns the results
|
||||
/// **unsorted** so the caller can apply its own sort criteria.
|
||||
pub fn build_model_fits(
|
||||
db: &ModelDatabase,
|
||||
specs: &SystemSpecs,
|
||||
installed: &InstalledIndex,
|
||||
context_limit: Option<u32>,
|
||||
forced_runtime: Option<InferenceRuntime>,
|
||||
) -> Vec<ModelFit> {
|
||||
use crate::fit::backend_compatible;
|
||||
|
||||
// Measured-throughput sources, most trustworthy first: the user's own
|
||||
// runs on this machine, llmfit community submissions recorded on
|
||||
// identical hardware, then localmaxxing medians on matching presets.
|
||||
let local_index = crate::share::LocalBenchIndex::load(specs);
|
||||
let community_index = crate::benchmarks::CommunityBenchIndex::for_specs(specs);
|
||||
let measured_index = crate::benchmarks::MeasuredTpsIndex::for_specs(specs);
|
||||
|
||||
let mut fits: Vec<ModelFit> = db
|
||||
.get_all_models()
|
||||
.iter()
|
||||
.filter(|m| backend_compatible(m, specs))
|
||||
.map(|m| {
|
||||
let mut fit =
|
||||
ModelFit::analyze_with_forced_runtime(m, specs, context_limit, forced_runtime);
|
||||
fit.installed = installed.is_installed(&m.name);
|
||||
fit.measured_tps = local_index
|
||||
.as_ref()
|
||||
.and_then(|idx| idx.lookup(&m.name))
|
||||
.or_else(|| community_index.as_ref().and_then(|idx| idx.lookup(&m.name)))
|
||||
.or_else(|| {
|
||||
measured_index
|
||||
.as_ref()
|
||||
.and_then(|idx| idx.lookup(&m.name, &fit.best_quant))
|
||||
});
|
||||
fit
|
||||
})
|
||||
.collect();
|
||||
apply_local_calibration(&mut fits);
|
||||
fits
|
||||
}
|
||||
|
||||
/// Calibrate formula estimates from benchmark runs made on this exact
|
||||
/// hardware: the user's own local runs, plus llmfit community submissions
|
||||
/// recorded on an identical configuration (so a fresh install benefits the
|
||||
/// moment anyone contributed on the same machine class).
|
||||
///
|
||||
/// Anchors must map to a catalog entry with a trustworthy size (>= 1B
|
||||
/// params, dense — MoE and tiny models don't scale like bandwidth-bound
|
||||
/// dense generation). The median measured/estimated ratio, clamped to
|
||||
/// [0.05, 3.0], scales every row's estimate and is recorded in
|
||||
/// `estimate_basis.local_calibration`.
|
||||
///
|
||||
/// Idempotent: ratios and scaling always derive from the uncalibrated
|
||||
/// estimate, so re-applying after a new bench never compounds.
|
||||
pub fn apply_local_calibration(fits: &mut [ModelFit]) {
|
||||
use crate::benchmarks::MeasuredSource;
|
||||
|
||||
fn uncalibrated(f: &ModelFit) -> f64 {
|
||||
match f.estimate_basis.local_calibration {
|
||||
Some(c) if c > 0.0 => f.estimated_tps / c,
|
||||
_ => f.estimated_tps,
|
||||
}
|
||||
}
|
||||
|
||||
let mut ratios: Vec<f64> = fits
|
||||
.iter()
|
||||
.filter(|f| f.model.params_b() >= 1.0 && !f.model.is_moe)
|
||||
.filter_map(|f| {
|
||||
let m = f.measured_tps.as_ref()?;
|
||||
if !matches!(
|
||||
m.source,
|
||||
MeasuredSource::LocalBench | MeasuredSource::CommunityLlmfit
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
let est = uncalibrated(f);
|
||||
(est > 0.0 && m.tok_s > 0.0).then(|| m.tok_s / est)
|
||||
})
|
||||
.collect();
|
||||
if ratios.is_empty() {
|
||||
return;
|
||||
}
|
||||
ratios.sort_by(|a, b| a.partial_cmp(b).expect("ratios are finite"));
|
||||
let factor = median(&ratios).clamp(0.05, 3.0);
|
||||
|
||||
for f in fits.iter_mut() {
|
||||
if f.estimated_tps <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
f.estimated_tps = uncalibrated(f) * factor;
|
||||
f.estimate_basis.local_calibration = Some(factor);
|
||||
}
|
||||
}
|
||||
|
||||
fn median(sorted: &[f64]) -> f64 {
|
||||
let n = sorted.len();
|
||||
if n % 2 == 1 {
|
||||
sorted[n / 2]
|
||||
} else {
|
||||
(sorted[n / 2 - 1] + sorted[n / 2]) / 2.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod calibration_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn median_of_sorted() {
|
||||
assert_eq!(median(&[0.1]), 0.1);
|
||||
assert_eq!(median(&[0.1, 0.3]), 0.2);
|
||||
assert_eq!(median(&[0.1, 0.2, 0.9]), 0.2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,786 @@
|
||||
//! LLM inference benchmarking against Ollama, vLLM, and MLX endpoints.
|
||||
//!
|
||||
//! Measures time-to-first-token (TTFT), tokens per second (TPS),
|
||||
//! and total latency using real inference requests.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Results from a single benchmark run.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct BenchRun {
|
||||
/// Time to first token in milliseconds, if measurable.
|
||||
/// - Ollama: measured from `eval_duration` (accurate).
|
||||
/// - vLLM/MLX: `None` — would require streaming to measure; only wall-clock
|
||||
/// total is available.
|
||||
pub ttft_ms: Option<f64>,
|
||||
/// Output tokens per second.
|
||||
pub tps: f64,
|
||||
/// Total request latency in milliseconds.
|
||||
pub total_ms: f64,
|
||||
/// Number of prompt tokens processed.
|
||||
pub prompt_tokens: u32,
|
||||
/// Number of output tokens generated.
|
||||
pub output_tokens: u32,
|
||||
}
|
||||
|
||||
/// Aggregated benchmark results across multiple runs.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct BenchResult {
|
||||
pub model: String,
|
||||
pub provider: String,
|
||||
pub runs: Vec<BenchRun>,
|
||||
pub summary: BenchSummary,
|
||||
}
|
||||
|
||||
/// Statistical summary of benchmark runs.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct BenchSummary {
|
||||
pub num_runs: usize,
|
||||
pub avg_ttft_ms: Option<f64>,
|
||||
pub avg_tps: f64,
|
||||
pub min_tps: f64,
|
||||
pub max_tps: f64,
|
||||
pub avg_total_ms: f64,
|
||||
pub avg_output_tokens: f64,
|
||||
}
|
||||
|
||||
impl BenchSummary {
|
||||
fn from_runs(runs: &[BenchRun]) -> Self {
|
||||
let n = runs.len() as f64;
|
||||
if runs.is_empty() {
|
||||
return BenchSummary {
|
||||
num_runs: 0,
|
||||
avg_ttft_ms: None,
|
||||
avg_tps: 0.0,
|
||||
min_tps: 0.0,
|
||||
max_tps: 0.0,
|
||||
avg_total_ms: 0.0,
|
||||
avg_output_tokens: 0.0,
|
||||
};
|
||||
}
|
||||
// Only compute avg TTFT if any run has a measured value
|
||||
let ttft_values: Vec<f64> = runs.iter().filter_map(|r| r.ttft_ms).collect();
|
||||
let avg_ttft_ms = if ttft_values.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ttft_values.iter().sum::<f64>() / ttft_values.len() as f64)
|
||||
};
|
||||
BenchSummary {
|
||||
num_runs: runs.len(),
|
||||
avg_ttft_ms,
|
||||
avg_tps: runs.iter().map(|r| r.tps).sum::<f64>() / n,
|
||||
min_tps: runs.iter().map(|r| r.tps).fold(f64::INFINITY, f64::min),
|
||||
max_tps: runs.iter().map(|r| r.tps).fold(0.0_f64, f64::max),
|
||||
avg_total_ms: runs.iter().map(|r| r.total_ms).sum::<f64>() / n,
|
||||
avg_output_tokens: runs.iter().map(|r| r.output_tokens as f64).sum::<f64>() / n,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test prompts of varying lengths for benchmarking.
|
||||
const BENCH_PROMPTS: &[&str] = &[
|
||||
"Explain what a hash table is in 2 sentences.",
|
||||
"Write a Python function that checks if a string is a palindrome. Include a docstring.",
|
||||
"Compare and contrast TCP and UDP protocols. Cover reliability, ordering, speed, and common use cases. Be concise.",
|
||||
"You are a senior software engineer. Review this code and suggest improvements:\n\n```python\ndef fib(n):\n if n <= 1:\n return n\n return fib(n-1) + fib(n-2)\n```",
|
||||
];
|
||||
|
||||
// ── Ollama benchmarking ────────────────────────────────────────────
|
||||
|
||||
/// Ollama /api/generate response fields we care about.
|
||||
/// Shared with `quality.rs` — both modules talk to the same endpoints.
|
||||
#[derive(serde::Deserialize, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub(crate) struct OllamaGenResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) response: String,
|
||||
#[serde(default)]
|
||||
pub(crate) eval_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(crate) eval_duration: Option<u64>, // nanoseconds
|
||||
#[serde(default)]
|
||||
pub(crate) prompt_eval_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(crate) prompt_eval_duration: Option<u64>, // nanoseconds
|
||||
#[serde(default)]
|
||||
pub(crate) total_duration: Option<u64>, // nanoseconds
|
||||
}
|
||||
|
||||
/// Benchmark a model via Ollama's /api/generate endpoint.
|
||||
pub fn bench_ollama(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
num_runs: usize,
|
||||
on_progress: &dyn Fn(usize, usize),
|
||||
) -> Result<BenchResult, String> {
|
||||
let url = format!("{}/api/generate", base_url.trim_end_matches('/'));
|
||||
let mut runs = Vec::with_capacity(num_runs);
|
||||
|
||||
// Warmup request (don't count it)
|
||||
on_progress(0, num_runs);
|
||||
if let Err(e) = ollama_generate(&url, model, "Say hello.", 300) {
|
||||
return Err(format!(
|
||||
"Warmup request failed (is the model loaded?): {}",
|
||||
e
|
||||
));
|
||||
}
|
||||
|
||||
for i in 0..num_runs {
|
||||
on_progress(i + 1, num_runs);
|
||||
let prompt = BENCH_PROMPTS[i % BENCH_PROMPTS.len()];
|
||||
let run = ollama_generate(&url, model, prompt, 300)?;
|
||||
runs.push(run);
|
||||
}
|
||||
|
||||
let summary = BenchSummary::from_runs(&runs);
|
||||
Ok(BenchResult {
|
||||
model: model.to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
runs,
|
||||
summary,
|
||||
})
|
||||
}
|
||||
|
||||
fn ollama_generate(
|
||||
url: &str,
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
max_tokens: u32,
|
||||
) -> Result<BenchRun, String> {
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": max_tokens,
|
||||
}
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = ureq::post(url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(300)))
|
||||
.build()
|
||||
.send_json(&body)
|
||||
.map_err(|e| format!("Ollama request failed: {}", e))?;
|
||||
|
||||
let total_wall = start.elapsed();
|
||||
|
||||
let resp_body: OllamaGenResponse = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("Ollama JSON parse error: {}", e))?;
|
||||
|
||||
// Ollama provides native timing in nanoseconds
|
||||
let prompt_tokens = resp_body.prompt_eval_count.unwrap_or(0) as u32;
|
||||
let output_tokens = resp_body.eval_count.unwrap_or(0) as u32;
|
||||
|
||||
let ttft_ms = resp_body
|
||||
.prompt_eval_duration
|
||||
.map(|ns| ns as f64 / 1_000_000.0);
|
||||
|
||||
let tps = if let (Some(eval_count), Some(eval_dur)) =
|
||||
(resp_body.eval_count, resp_body.eval_duration)
|
||||
{
|
||||
if eval_dur > 0 {
|
||||
eval_count as f64 / (eval_dur as f64 / 1_000_000_000.0)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
} else if output_tokens > 0 {
|
||||
// Fallback to wall-clock
|
||||
output_tokens as f64 / total_wall.as_secs_f64()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let total_ms = resp_body
|
||||
.total_duration
|
||||
.map(|ns| ns as f64 / 1_000_000.0)
|
||||
.unwrap_or(total_wall.as_secs_f64() * 1000.0);
|
||||
|
||||
Ok(BenchRun {
|
||||
ttft_ms,
|
||||
tps,
|
||||
total_ms,
|
||||
prompt_tokens,
|
||||
output_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
// ── OpenAI-compatible benchmarking (vLLM, MLX) ────────────────────
|
||||
|
||||
/// OpenAI-compatible chat completion response fields we care about.
|
||||
/// Shared with `quality.rs` — both modules talk to the same endpoints.
|
||||
#[derive(serde::Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub(crate) struct ChatCompletionResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) choices: Vec<ChatChoice>,
|
||||
#[serde(default)]
|
||||
pub(crate) usage: Option<ChatUsage>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub(crate) struct ChatChoice {
|
||||
#[serde(default)]
|
||||
pub(crate) message: Option<ChatMessage>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub(crate) struct ChatMessage {
|
||||
#[serde(default)]
|
||||
pub(crate) content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct ChatUsage {
|
||||
#[serde(default)]
|
||||
pub(crate) prompt_tokens: u32,
|
||||
#[serde(default)]
|
||||
pub(crate) completion_tokens: u32,
|
||||
}
|
||||
|
||||
/// Benchmark a model via OpenAI-compatible /v1/chat/completions (vLLM, MLX).
|
||||
pub fn bench_openai_compat(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
provider_name: &str,
|
||||
num_runs: usize,
|
||||
on_progress: &dyn Fn(usize, usize),
|
||||
) -> Result<BenchResult, String> {
|
||||
let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
|
||||
let mut runs = Vec::with_capacity(num_runs);
|
||||
|
||||
// Warmup
|
||||
on_progress(0, num_runs);
|
||||
if let Err(e) = openai_chat(&url, model, "Say hello.", 100) {
|
||||
return Err(format!(
|
||||
"Warmup request failed (is the endpoint reachable?): {}",
|
||||
e
|
||||
));
|
||||
}
|
||||
|
||||
for i in 0..num_runs {
|
||||
on_progress(i + 1, num_runs);
|
||||
let prompt = BENCH_PROMPTS[i % BENCH_PROMPTS.len()];
|
||||
let run = openai_chat(&url, model, prompt, 300)?;
|
||||
runs.push(run);
|
||||
}
|
||||
|
||||
let summary = BenchSummary::from_runs(&runs);
|
||||
Ok(BenchResult {
|
||||
model: model.to_string(),
|
||||
provider: provider_name.to_string(),
|
||||
runs,
|
||||
summary,
|
||||
})
|
||||
}
|
||||
|
||||
fn openai_chat(url: &str, model: &str, prompt: &str, max_tokens: u32) -> Result<BenchRun, String> {
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"stream": false,
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// TTFT is estimated from wall clock: prompt_tokens / total_tokens * total_time.
|
||||
// This is a rough heuristic — actual TTFT requires streaming (not implemented).
|
||||
let resp = ureq::post(url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(300)))
|
||||
.build()
|
||||
.send_json(&body)
|
||||
.map_err(|e| format!("{} request failed: {}", url, e))?;
|
||||
|
||||
let total_wall = start.elapsed();
|
||||
|
||||
let completion: ChatCompletionResponse = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("JSON parse error: {}", e))?;
|
||||
|
||||
let usage = completion.usage.unwrap_or(ChatUsage {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
});
|
||||
|
||||
let output_tokens = usage.completion_tokens;
|
||||
let prompt_tokens = usage.prompt_tokens;
|
||||
|
||||
// TTFT cannot be measured without streaming — set to None.
|
||||
let total_ms = total_wall.as_secs_f64() * 1000.0;
|
||||
|
||||
let tps = if output_tokens > 0 && total_wall.as_secs_f64() > 0.0 {
|
||||
output_tokens as f64 / total_wall.as_secs_f64()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(BenchRun {
|
||||
ttft_ms: None,
|
||||
tps,
|
||||
total_ms,
|
||||
prompt_tokens,
|
||||
output_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Auto-detect and benchmark ──────────────────────────────────────
|
||||
|
||||
/// Which provider to benchmark against.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BenchTarget {
|
||||
Ollama { url: String, model: String },
|
||||
VLlm { url: String, model: String },
|
||||
Mlx { url: String, model: String },
|
||||
LlamaCpp { url: String, model: String },
|
||||
}
|
||||
|
||||
/// Base URL for a running llama-server instance.
|
||||
/// `LLAMA_SERVER_HOST` (full URL) wins; otherwise localhost with
|
||||
/// `LLAMA_SERVER_PORT` (default 8080, llama-server's own default).
|
||||
pub fn llamacpp_url() -> String {
|
||||
if let Ok(host) = std::env::var("LLAMA_SERVER_HOST")
|
||||
&& !host.trim().is_empty()
|
||||
{
|
||||
return host.trim().trim_end_matches('/').to_string();
|
||||
}
|
||||
let port = std::env::var("LLAMA_SERVER_PORT").unwrap_or_else(|_| "8080".to_string());
|
||||
format!("http://localhost:{}", port)
|
||||
}
|
||||
|
||||
/// Positively identify a llama-server instance via its `/props` endpoint.
|
||||
/// llama.cpp serves it; MLX and vLLM return 404, which disambiguates
|
||||
/// llama-server from `mlx_lm.server` on the shared default port 8080.
|
||||
pub fn probe_llamacpp(base_url: &str) -> bool {
|
||||
ureq::get(&format!("{}/props", base_url.trim_end_matches('/')))
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(2)))
|
||||
.build()
|
||||
.call()
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Auto-detect available providers and pick the best one to benchmark.
|
||||
pub fn auto_detect_target(model_hint: Option<&str>) -> Result<BenchTarget, String> {
|
||||
// Check vLLM via VLLM_PORT env var (defaults to 8000)
|
||||
let vllm_port = std::env::var("VLLM_PORT").unwrap_or_else(|_| "8000".to_string());
|
||||
let vllm_url = format!("http://localhost:{}", vllm_port);
|
||||
if let Ok(model_name) = detect_vllm_model(&vllm_url, model_hint) {
|
||||
return Ok(BenchTarget::VLlm {
|
||||
url: vllm_url,
|
||||
model: model_name,
|
||||
});
|
||||
}
|
||||
|
||||
// Check Ollama
|
||||
let ollama_url =
|
||||
std::env::var("OLLAMA_HOST").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
if ureq::get(&format!("{}/api/tags", ollama_url))
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(2)))
|
||||
.build()
|
||||
.call()
|
||||
.is_ok()
|
||||
&& let Ok(model_name) = detect_ollama_model(&ollama_url, model_hint)
|
||||
{
|
||||
return Ok(BenchTarget::Ollama {
|
||||
url: ollama_url,
|
||||
model: model_name,
|
||||
});
|
||||
}
|
||||
|
||||
// Check llama-server before MLX: both default to port 8080, but only
|
||||
// llama.cpp answers /props, so it can be identified positively.
|
||||
let llama_url = llamacpp_url();
|
||||
if probe_llamacpp(&llama_url)
|
||||
&& let Ok(model_name) = detect_openai_model(&llama_url, model_hint)
|
||||
{
|
||||
return Ok(BenchTarget::LlamaCpp {
|
||||
url: llama_url,
|
||||
model: model_name,
|
||||
});
|
||||
}
|
||||
|
||||
// Check MLX
|
||||
let mlx_url =
|
||||
std::env::var("MLX_LM_HOST").unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||
if ureq::get(&format!("{}/v1/models", mlx_url))
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(2)))
|
||||
.build()
|
||||
.call()
|
||||
.is_ok()
|
||||
&& let Ok(model_name) = detect_openai_model(&mlx_url, model_hint)
|
||||
{
|
||||
return Ok(BenchTarget::Mlx {
|
||||
url: mlx_url,
|
||||
model: model_name,
|
||||
});
|
||||
}
|
||||
|
||||
Err("No inference provider found. Start Ollama, vLLM, MLX, or llama-server first.".to_string())
|
||||
}
|
||||
|
||||
/// Discover all available models across all providers.
|
||||
pub fn discover_all_targets() -> Vec<BenchTarget> {
|
||||
let mut targets = Vec::new();
|
||||
|
||||
// Check vLLM via VLLM_PORT env var (defaults to 8000)
|
||||
let vllm_port = std::env::var("VLLM_PORT").unwrap_or_else(|_| "8000".to_string());
|
||||
let vllm_url = format!("http://localhost:{}", vllm_port);
|
||||
if let Ok(models) = list_openai_models(&vllm_url) {
|
||||
for model in models {
|
||||
targets.push(BenchTarget::VLlm {
|
||||
url: vllm_url.clone(),
|
||||
model,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check Ollama
|
||||
let ollama_url =
|
||||
std::env::var("OLLAMA_HOST").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
if let Ok(models) = list_ollama_models(&ollama_url) {
|
||||
for model in models {
|
||||
targets.push(BenchTarget::Ollama {
|
||||
url: ollama_url.clone(),
|
||||
model,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check llama-server before MLX: both default to port 8080, but only
|
||||
// llama.cpp answers /props, so it can be identified positively.
|
||||
let llama_url = llamacpp_url();
|
||||
let llamacpp_found = probe_llamacpp(&llama_url);
|
||||
if llamacpp_found && let Ok(models) = list_openai_models(&llama_url) {
|
||||
for model in models {
|
||||
targets.push(BenchTarget::LlamaCpp {
|
||||
url: llama_url.clone(),
|
||||
model,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check MLX (skip if the llama-server probe already claimed this URL,
|
||||
// e.g. both on the default port 8080)
|
||||
let mlx_url =
|
||||
std::env::var("MLX_LM_HOST").unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||
if !(llamacpp_found && mlx_url.trim_end_matches('/') == llama_url)
|
||||
&& let Ok(models) = list_openai_models(&mlx_url)
|
||||
{
|
||||
for model in models {
|
||||
targets.push(BenchTarget::Mlx {
|
||||
url: mlx_url.clone(),
|
||||
model,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
fn list_openai_models(base_url: &str) -> Result<Vec<String>, String> {
|
||||
let url = format!("{}/v1/models", base_url);
|
||||
let resp = ureq::get(&url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(3)))
|
||||
.build()
|
||||
.call()
|
||||
.map_err(|e| format!("{}", e))?;
|
||||
|
||||
let body: serde_json::Value = resp.into_body().read_json().map_err(|e| format!("{}", e))?;
|
||||
let models = body
|
||||
.get("data")
|
||||
.and_then(|d: &serde_json::Value| d.as_array())
|
||||
.ok_or("no data")?;
|
||||
|
||||
Ok(models
|
||||
.iter()
|
||||
.filter_map(|m| {
|
||||
m.get("id")
|
||||
.and_then(|i: &serde_json::Value| i.as_str())
|
||||
.map(|s| s.to_string())
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn list_ollama_models(base_url: &str) -> Result<Vec<String>, String> {
|
||||
let url = format!("{}/api/tags", base_url);
|
||||
let resp = ureq::get(&url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(3)))
|
||||
.build()
|
||||
.call()
|
||||
.map_err(|e| format!("{}", e))?;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Tags {
|
||||
models: Vec<M>,
|
||||
}
|
||||
#[derive(serde::Deserialize)]
|
||||
struct M {
|
||||
name: String,
|
||||
}
|
||||
|
||||
let tags: Tags = resp.into_body().read_json().map_err(|e| format!("{}", e))?;
|
||||
Ok(tags.models.into_iter().map(|m| m.name).collect())
|
||||
}
|
||||
|
||||
/// Detect model from a given base URL (OpenAI-compatible /v1/models).
|
||||
pub fn detect_model_from_url(base_url: &str, hint: Option<&str>) -> Result<String, String> {
|
||||
detect_openai_model(base_url, hint)
|
||||
}
|
||||
|
||||
fn detect_vllm_model(base_url: &str, hint: Option<&str>) -> Result<String, String> {
|
||||
detect_openai_model(base_url, hint)
|
||||
}
|
||||
|
||||
fn detect_openai_model(base_url: &str, hint: Option<&str>) -> Result<String, String> {
|
||||
let url = format!("{}/v1/models", base_url);
|
||||
let resp = ureq::get(&url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(5)))
|
||||
.build()
|
||||
.call()
|
||||
.map_err(|e| format!("Cannot reach {}: {}", url, e))?;
|
||||
|
||||
let body: serde_json::Value = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("JSON error: {}", e))?;
|
||||
|
||||
let models = body
|
||||
.get("data")
|
||||
.and_then(|d: &serde_json::Value| d.as_array())
|
||||
.ok_or("No models found")?;
|
||||
|
||||
if models.is_empty() {
|
||||
return Err("No models loaded".to_string());
|
||||
}
|
||||
|
||||
// If hint provided, find matching model
|
||||
if let Some(hint) = hint {
|
||||
let hint_lower = hint.to_lowercase();
|
||||
for m in models {
|
||||
if let Some(id) = m.get("id").and_then(|i: &serde_json::Value| i.as_str())
|
||||
&& id.to_lowercase().contains(&hint_lower)
|
||||
{
|
||||
return Ok(id.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return first model
|
||||
models[0]
|
||||
.get("id")
|
||||
.and_then(|i: &serde_json::Value| i.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or("Model has no id".to_string())
|
||||
}
|
||||
|
||||
fn detect_ollama_model(base_url: &str, hint: Option<&str>) -> Result<String, String> {
|
||||
let url = format!("{}/api/tags", base_url);
|
||||
let resp = ureq::get(&url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(5)))
|
||||
.build()
|
||||
.call()
|
||||
.map_err(|e| format!("Cannot reach Ollama: {}", e))?;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Tags {
|
||||
models: Vec<Model>,
|
||||
}
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Model {
|
||||
name: String,
|
||||
}
|
||||
|
||||
let tags: Tags = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("JSON error: {}", e))?;
|
||||
|
||||
if tags.models.is_empty() {
|
||||
return Err("No models installed in Ollama".to_string());
|
||||
}
|
||||
|
||||
if let Some(hint) = hint {
|
||||
let hint_lower = hint.to_lowercase();
|
||||
for m in &tags.models {
|
||||
if m.name.to_lowercase().contains(&hint_lower) {
|
||||
return Ok(m.name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(tags.models[0].name.clone())
|
||||
}
|
||||
|
||||
// ── Display helpers ────────────────────────────────────────────────
|
||||
|
||||
impl BenchResult {
|
||||
pub fn display(&self) {
|
||||
println!();
|
||||
println!(" === Benchmark Results ===");
|
||||
println!(" Model: {}", self.model);
|
||||
println!(" Provider: {}", self.provider);
|
||||
println!(" Runs: {}", self.summary.num_runs);
|
||||
println!();
|
||||
println!(
|
||||
" TPS: {:.1} avg ({:.1} min / {:.1} max)",
|
||||
self.summary.avg_tps, self.summary.min_tps, self.summary.max_tps
|
||||
);
|
||||
if let Some(ttft) = self.summary.avg_ttft_ms {
|
||||
println!(" TTFT: {:.0} ms avg", ttft);
|
||||
} else {
|
||||
println!(" TTFT: n/a (streaming required)");
|
||||
}
|
||||
println!(" Latency: {:.0} ms avg", self.summary.avg_total_ms);
|
||||
println!(
|
||||
" Output: {:.0} tokens avg",
|
||||
self.summary.avg_output_tokens
|
||||
);
|
||||
println!();
|
||||
|
||||
// Per-run breakdown
|
||||
println!(" Run TPS TTFT Latency Tokens");
|
||||
println!(" ─── ─────── ─────── ─────── ──────");
|
||||
for (i, run) in self.runs.iter().enumerate() {
|
||||
println!(
|
||||
" {:>3} {:>6.1} {:>5}ms {:>5.0}ms {:>5}",
|
||||
i + 1,
|
||||
run.tps,
|
||||
run.ttft_ms
|
||||
.map(|t| format!("{:.0}", t))
|
||||
.unwrap_or_else(|| "n/a".to_string()),
|
||||
run.total_ms,
|
||||
run.output_tokens
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
pub fn display_json(&self) {
|
||||
let json = serde_json::json!({
|
||||
"benchmark": {
|
||||
"model": self.model,
|
||||
"provider": self.provider,
|
||||
"summary": self.summary,
|
||||
"runs": self.runs,
|
||||
}
|
||||
});
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json).expect("JSON serialization failed")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_run(ttft_ms: f64, tps: f64, total_ms: f64, output_tokens: u32) -> BenchRun {
|
||||
BenchRun {
|
||||
ttft_ms: Some(ttft_ms),
|
||||
tps,
|
||||
total_ms,
|
||||
prompt_tokens: 10,
|
||||
output_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────
|
||||
// BenchSummary::from_runs
|
||||
// ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_summary_multiple_runs() {
|
||||
let runs = vec![
|
||||
make_run(100.0, 20.0, 500.0, 50),
|
||||
make_run(150.0, 30.0, 600.0, 60),
|
||||
make_run(200.0, 10.0, 700.0, 70),
|
||||
];
|
||||
let s = BenchSummary::from_runs(&runs);
|
||||
|
||||
assert_eq!(s.num_runs, 3);
|
||||
assert!((s.avg_ttft_ms.unwrap() - 150.0).abs() < 0.01);
|
||||
assert!((s.avg_tps - 20.0).abs() < 0.01);
|
||||
assert!((s.min_tps - 10.0).abs() < 0.01);
|
||||
assert!((s.max_tps - 30.0).abs() < 0.01);
|
||||
assert!((s.avg_total_ms - 600.0).abs() < 0.01);
|
||||
assert!((s.avg_output_tokens - 60.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_summary_single_run() {
|
||||
let runs = vec![make_run(100.0, 25.0, 500.0, 50)];
|
||||
let s = BenchSummary::from_runs(&runs);
|
||||
|
||||
assert_eq!(s.num_runs, 1);
|
||||
assert!((s.avg_ttft_ms.unwrap() - 100.0).abs() < 0.01);
|
||||
assert!((s.avg_tps - 25.0).abs() < 0.01);
|
||||
// min == max == avg for a single run
|
||||
assert!((s.min_tps - 25.0).abs() < 0.01);
|
||||
assert!((s.max_tps - 25.0).abs() < 0.01);
|
||||
assert!((s.avg_total_ms - 500.0).abs() < 0.01);
|
||||
assert!((s.avg_output_tokens - 50.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_summary_empty_runs() {
|
||||
let runs: Vec<BenchRun> = vec![];
|
||||
let s = BenchSummary::from_runs(&runs);
|
||||
|
||||
assert_eq!(s.num_runs, 0);
|
||||
assert_eq!(s.avg_tps, 0.0);
|
||||
assert_eq!(s.min_tps, 0.0);
|
||||
assert_eq!(s.max_tps, 0.0);
|
||||
assert_eq!(s.avg_ttft_ms, None);
|
||||
assert_eq!(s.avg_total_ms, 0.0);
|
||||
assert_eq!(s.avg_output_tokens, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_summary_min_max_correctness() {
|
||||
let runs = vec![
|
||||
make_run(50.0, 5.0, 200.0, 20),
|
||||
make_run(60.0, 50.0, 300.0, 30),
|
||||
make_run(70.0, 25.0, 400.0, 40),
|
||||
make_run(80.0, 100.0, 500.0, 50),
|
||||
make_run(90.0, 1.0, 600.0, 60),
|
||||
];
|
||||
let s = BenchSummary::from_runs(&runs);
|
||||
|
||||
assert_eq!(s.num_runs, 5);
|
||||
assert!((s.min_tps - 1.0).abs() < 0.01);
|
||||
assert!((s.max_tps - 100.0).abs() < 0.01);
|
||||
// avg_tps = (5+50+25+100+1)/5 = 36.2
|
||||
assert!((s.avg_tps - 36.2).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_summary_identical_runs() {
|
||||
let runs = vec![
|
||||
make_run(100.0, 20.0, 500.0, 50),
|
||||
make_run(100.0, 20.0, 500.0, 50),
|
||||
make_run(100.0, 20.0, 500.0, 50),
|
||||
];
|
||||
let s = BenchSummary::from_runs(&runs);
|
||||
|
||||
assert_eq!(s.num_runs, 3);
|
||||
assert!((s.avg_tps - 20.0).abs() < 0.01);
|
||||
assert!((s.min_tps - 20.0).abs() < 0.01);
|
||||
assert!((s.max_tps - 20.0).abs() < 0.01);
|
||||
assert!((s.avg_ttft_ms.unwrap() - 100.0).abs() < 0.01);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,364 @@
|
||||
//! Kubernetes DRA claim generation.
|
||||
//!
|
||||
//! Turns a (model, quantization, min-tok/s) target into a ResourceClaim or
|
||||
//! ResourceClaimTemplate whose CEL selector encodes the fit inequality
|
||||
//! against attributes published by the llmfit.ai DRA driver (llmfit-dra).
|
||||
//! The driver publishes physics inputs (memory capacity, bandwidth); this
|
||||
//! module inlines the model-specific constants from the database so the
|
||||
//! kube-scheduler can evaluate fit at claim time. Nothing here runs in the
|
||||
//! serving path — the output is plain YAML for kubectl/GitOps.
|
||||
|
||||
use crate::models::LlmModel;
|
||||
|
||||
/// Attribute/driver domain used by llmfit-dra.
|
||||
pub const DRIVER_DOMAIN: &str = "llmfit.ai";
|
||||
|
||||
/// KV-cache / runtime headroom multiplier applied when the requested quant
|
||||
/// differs from the database entry (whose min_vram_gb already includes it).
|
||||
const WEIGHTS_HEADROOM: f64 = 1.2;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClaimTarget {
|
||||
pub min_tps: f64,
|
||||
/// Backend efficiency as a percentage (fit.rs default_efficiency = 0.55).
|
||||
pub efficiency_pct: u32,
|
||||
pub device_class: String,
|
||||
/// Emit a ResourceClaimTemplate (for pod templates) instead of a bare
|
||||
/// ResourceClaim.
|
||||
pub template: bool,
|
||||
/// Override the database entry's quantization.
|
||||
pub quant: Option<String>,
|
||||
/// Override the generated metadata.name.
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for ClaimTarget {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
min_tps: 20.0,
|
||||
efficiency_pct: 55,
|
||||
device_class: DRIVER_DOMAIN.to_string(),
|
||||
template: false,
|
||||
quant: None,
|
||||
name: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The two constants the CEL selector needs.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct FitBounds {
|
||||
/// Device memory floor, binary gibibytes (weights + headroom).
|
||||
pub memory_gi: u64,
|
||||
/// Bandwidth floor in GB/s such that bw × efficiency / weights ≥ min_tps.
|
||||
pub min_bandwidth_gbs: u64,
|
||||
/// Weights size used for the bound, for provenance comments.
|
||||
pub weights_gb: f64,
|
||||
pub quant: String,
|
||||
}
|
||||
|
||||
pub fn fit_bounds(model: &LlmModel, target: &ClaimTarget) -> Result<FitBounds, String> {
|
||||
if target.min_tps <= 0.0 {
|
||||
return Err("--min-tps must be > 0".to_string());
|
||||
}
|
||||
if target.efficiency_pct == 0 || target.efficiency_pct > 100 {
|
||||
return Err("--efficiency must be in 1..=100".to_string());
|
||||
}
|
||||
let quant = target
|
||||
.quant
|
||||
.clone()
|
||||
.unwrap_or_else(|| model.quantization.clone());
|
||||
let weights_gb = model.estimate_disk_gb(&quant);
|
||||
if weights_gb <= 0.0 {
|
||||
return Err(format!(
|
||||
"cannot size model '{}' (unknown parameter count)",
|
||||
model.name
|
||||
));
|
||||
}
|
||||
// Memory floor: the database's min_vram_gb is authoritative for the
|
||||
// entry's own quant (it already includes KV/runtime headroom); for a
|
||||
// quant override, fall back to weights × headroom.
|
||||
let memory_gb = if quant == model.quantization {
|
||||
model
|
||||
.min_vram_gb
|
||||
.unwrap_or(model.min_ram_gb)
|
||||
.max(weights_gb)
|
||||
} else {
|
||||
weights_gb * WEIGHTS_HEADROOM
|
||||
};
|
||||
// tok/s ≈ bandwidth × efficiency / weights ⇒ bandwidth ≥ tps × weights / eff
|
||||
let min_bw = target.min_tps * weights_gb * 100.0 / f64::from(target.efficiency_pct);
|
||||
Ok(FitBounds {
|
||||
memory_gi: memory_gb.ceil() as u64,
|
||||
min_bandwidth_gbs: min_bw.ceil() as u64,
|
||||
weights_gb,
|
||||
quant,
|
||||
})
|
||||
}
|
||||
|
||||
/// DNS-label-safe name derived from the model name, e.g.
|
||||
/// "Qwen2.5 32B Instruct" → "qwen2-5-32b-instruct-fit".
|
||||
pub fn claim_name(model: &LlmModel) -> String {
|
||||
let mut s: String = model
|
||||
.name
|
||||
.to_lowercase()
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
|
||||
.collect();
|
||||
while s.contains("--") {
|
||||
s = s.replace("--", "-");
|
||||
}
|
||||
let s = s.trim_matches('-');
|
||||
let mut base = s.chars().take(48).collect::<String>();
|
||||
base = base.trim_matches('-').to_string();
|
||||
format!("{base}-fit")
|
||||
}
|
||||
|
||||
/// Render the resolved fit bounds as machine-readable JSON, for programmatic
|
||||
/// consumers (the llmfit-dra ModelClaim controller renders its own
|
||||
/// ResourceClaimTemplate from these numbers instead of scraping YAML).
|
||||
/// `resolver_version` is the binary version, recorded so consumers can
|
||||
/// re-resolve when the model database advances.
|
||||
pub fn render_json(
|
||||
model: &LlmModel,
|
||||
target: &ClaimTarget,
|
||||
resolver_version: &str,
|
||||
) -> Result<String, String> {
|
||||
let b = fit_bounds(model, target)?;
|
||||
let name = target.name.clone().unwrap_or_else(|| claim_name(model));
|
||||
let out = serde_json::json!({
|
||||
"model": model.name,
|
||||
"claimName": name,
|
||||
"quant": b.quant,
|
||||
"weightsGb": (b.weights_gb * 10.0).round() / 10.0,
|
||||
"memoryGi": b.memory_gi,
|
||||
"minBandwidthGBs": b.min_bandwidth_gbs,
|
||||
"minTps": target.min_tps,
|
||||
"efficiencyPct": target.efficiency_pct,
|
||||
"deviceClass": target.device_class,
|
||||
"resolverVersion": resolver_version,
|
||||
});
|
||||
serde_json::to_string_pretty(&out).map_err(|e| format!("JSON serialization failed: {e}"))
|
||||
}
|
||||
|
||||
/// Render the claim YAML. Built as a template string (not serde) so the
|
||||
/// output carries provenance comments explaining where every constant came
|
||||
/// from — the file is meant to be committed to GitOps repos and read by
|
||||
/// humans.
|
||||
pub fn render(model: &LlmModel, target: &ClaimTarget) -> Result<String, String> {
|
||||
let b = fit_bounds(model, target)?;
|
||||
let name = target.name.clone().unwrap_or_else(|| claim_name(model));
|
||||
let eff = target.efficiency_pct;
|
||||
let ind = if target.template { " " } else { " " };
|
||||
// Continuation lines must sit at exactly the block scalar's content
|
||||
// indentation (first line = ind + 16) for clean YAML folding.
|
||||
let pad = format!("{ind} ");
|
||||
// Every optional lookup is guarded with CEL map membership: a missing
|
||||
// attribute must mean "device does not match", not a CEL runtime error.
|
||||
// Unguarded access errors on any device without the attribute (the cpu0
|
||||
// fallback, unindexed virtual display adapters on servers with BMC
|
||||
// framebuffers) and can wrongly disqualify allocations.
|
||||
let cel = format!(
|
||||
"'memory' in device.capacity['{d}'] &&\n\
|
||||
{pad}device.capacity['{d}'].memory.compareTo(quantity('{mem}Gi')) >= 0 &&\n\
|
||||
{pad}'memoryBandwidthGBs' in device.attributes['{d}'] &&\n\
|
||||
{pad}device.attributes['{d}'].memoryBandwidthGBs >= {bw} &&\n\
|
||||
{pad}'healthy' in device.attributes['{d}'] &&\n\
|
||||
{pad}device.attributes['{d}'].healthy",
|
||||
d = DRIVER_DOMAIN,
|
||||
mem = b.memory_gi,
|
||||
bw = b.min_bandwidth_gbs,
|
||||
);
|
||||
let header = format!(
|
||||
"# Generated by llmfit claim — do not compute these constants by hand.\n\
|
||||
# model: {name} ({params:.1}B params, {quant} ≈ {weights:.1} GB weights)\n\
|
||||
# fit: tok/s ≈ bandwidth × {eff}% / {weights:.1} GB ⇒ bandwidth ≥ {bw} GB/s for ≥ {tps} tok/s\n\
|
||||
# memory: ≥ {mem} Gi (weights + KV/runtime headroom)\n",
|
||||
name = model.name,
|
||||
params = model.params_b(),
|
||||
quant = b.quant,
|
||||
weights = b.weights_gb,
|
||||
eff = eff,
|
||||
bw = b.min_bandwidth_gbs,
|
||||
tps = target.min_tps,
|
||||
mem = b.memory_gi,
|
||||
);
|
||||
let devices = format!(
|
||||
"devices:\n\
|
||||
{i} requests:\n\
|
||||
{i} - name: model\n\
|
||||
{i} exactly:\n\
|
||||
{i} deviceClassName: {class}\n\
|
||||
{i} selectors:\n\
|
||||
{i} - cel:\n\
|
||||
{i} expression: >-\n\
|
||||
{i} {cel}",
|
||||
i = ind,
|
||||
class = target.device_class,
|
||||
cel = cel,
|
||||
);
|
||||
let body = if target.template {
|
||||
format!(
|
||||
"apiVersion: resource.k8s.io/v1\n\
|
||||
kind: ResourceClaimTemplate\n\
|
||||
metadata:\n\
|
||||
\x20 name: {name}\n\
|
||||
spec:\n\
|
||||
\x20 spec:\n\
|
||||
\x20 {devices}\n"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"apiVersion: resource.k8s.io/v1\n\
|
||||
kind: ResourceClaim\n\
|
||||
metadata:\n\
|
||||
\x20 name: {name}\n\
|
||||
spec:\n\
|
||||
\x20 {devices}\n"
|
||||
)
|
||||
};
|
||||
Ok(format!("{header}{body}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn model(quant: &str, min_vram: Option<f64>) -> LlmModel {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"name": "Test Model 7B",
|
||||
"provider": "test",
|
||||
"parameter_count": "7B",
|
||||
"parameters_raw": 7_000_000_000u64,
|
||||
"min_ram_gb": 8.0,
|
||||
"recommended_ram_gb": 10.0,
|
||||
"min_vram_gb": min_vram,
|
||||
"quantization": quant,
|
||||
"context_length": 8192,
|
||||
"use_case": "general",
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_json_golden() {
|
||||
let json = render_json(
|
||||
&model("Q4_K_M", Some(6.0)),
|
||||
&ClaimTarget::default(),
|
||||
"9.9.9-test",
|
||||
)
|
||||
.unwrap();
|
||||
let v: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(v["model"], "Test Model 7B");
|
||||
assert_eq!(v["claimName"], "test-model-7b-fit");
|
||||
assert_eq!(v["quant"], "Q4_K_M");
|
||||
assert_eq!(v["memoryGi"], 6);
|
||||
assert_eq!(v["minBandwidthGBs"], 148);
|
||||
assert_eq!(v["minTps"], 20.0);
|
||||
assert_eq!(v["efficiencyPct"], 55);
|
||||
assert_eq!(v["deviceClass"], "llmfit.ai");
|
||||
assert_eq!(v["resolverVersion"], "9.9.9-test");
|
||||
// weightsGb rounded to one decimal
|
||||
assert!(v["weightsGb"].as_f64().unwrap() > 3.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_json_propagates_resolution_errors() {
|
||||
let mut t = ClaimTarget::default();
|
||||
t.min_tps = 0.0;
|
||||
assert!(render_json(&model("Q4_K_M", None), &t, "x").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_use_db_memory_for_entry_quant() {
|
||||
// Q4_K_M: 7B × 0.58 bpp = 4.06 GB weights; db min_vram 6.0 wins.
|
||||
let b = fit_bounds(&model("Q4_K_M", Some(6.0)), &ClaimTarget::default()).unwrap();
|
||||
assert_eq!(b.memory_gi, 6);
|
||||
// bw ≥ 20 × 4.06 / 0.55 = 147.6… → 148
|
||||
assert_eq!(b.min_bandwidth_gbs, 148);
|
||||
assert_eq!(b.quant, "Q4_K_M");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_apply_headroom_on_quant_override() {
|
||||
let t = ClaimTarget {
|
||||
quant: Some("Q8_0".to_string()),
|
||||
..ClaimTarget::default()
|
||||
};
|
||||
// Q8_0: 7B × 1.05 bpp = 7.35 GB weights; ×1.2 headroom = 8.82 → 9 Gi.
|
||||
let b = fit_bounds(&model("Q4_K_M", Some(6.0)), &t).unwrap();
|
||||
assert_eq!(b.memory_gi, 9);
|
||||
assert_eq!(b.min_bandwidth_gbs, 268); // 20 × 7.35 / 0.55 = 267.3…
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_reject_nonsense() {
|
||||
assert!(
|
||||
fit_bounds(
|
||||
&model("Q4_K_M", None),
|
||||
&ClaimTarget {
|
||||
min_tps: 0.0,
|
||||
..ClaimTarget::default()
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
fit_bounds(
|
||||
&model("Q4_K_M", None),
|
||||
&ClaimTarget {
|
||||
efficiency_pct: 0,
|
||||
..ClaimTarget::default()
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_is_dns_label_safe() {
|
||||
let mut m = model("Q4_K_M", None);
|
||||
m.name = "Qwen2.5 32B Instruct".to_string();
|
||||
assert_eq!(claim_name(&m), "qwen2-5-32b-instruct-fit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_claim_yaml_shape() {
|
||||
let y = render(&model("Q4_K_M", Some(6.0)), &ClaimTarget::default()).unwrap();
|
||||
assert!(y.contains("kind: ResourceClaim\n"));
|
||||
assert!(y.contains("name: test-model-7b-fit"));
|
||||
assert!(y.contains("deviceClassName: llmfit.ai"));
|
||||
assert!(y.contains("quantity('6Gi')"));
|
||||
assert!(y.contains("memoryBandwidthGBs >= 148"));
|
||||
assert!(y.contains(".healthy"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_guards_every_optional_lookup() {
|
||||
// Missing attributes must be a non-match, not a CEL runtime error:
|
||||
// each capacity/attribute access is preceded by an `in` guard.
|
||||
let y = render(&model("Q4_K_M", Some(6.0)), &ClaimTarget::default()).unwrap();
|
||||
for guard in [
|
||||
"'memory' in device.capacity['llmfit.ai']",
|
||||
"'memoryBandwidthGBs' in device.attributes['llmfit.ai']",
|
||||
"'healthy' in device.attributes['llmfit.ai']",
|
||||
] {
|
||||
assert!(y.contains(guard), "missing guard: {guard}\n{y}");
|
||||
}
|
||||
// Guard must appear before the corresponding access.
|
||||
let mem_guard = y.find("'memory' in").unwrap();
|
||||
let mem_access = y.find(".memory.compareTo").unwrap();
|
||||
assert!(mem_guard < mem_access);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_template_wraps_spec() {
|
||||
let t = ClaimTarget {
|
||||
template: true,
|
||||
..ClaimTarget::default()
|
||||
};
|
||||
let y = render(&model("Q4_K_M", None), &t).unwrap();
|
||||
assert!(y.contains("kind: ResourceClaimTemplate\n"));
|
||||
assert!(y.contains("spec:\n spec:\n"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
//! Hardware diagnostic dump for bug reports.
|
||||
//!
|
||||
//! `llmfit doctor` captures the raw output of every external tool the GPU
|
||||
//! detection paths in [`crate::hardware`] shell out to, alongside what llmfit
|
||||
//! actually detected. Users paste the dump into GitHub issues; each report
|
||||
//! then doubles as a parser regression fixture (the verbatim tool output can
|
||||
//! be dropped straight into `hardware.rs` tests).
|
||||
|
||||
use crate::hardware::SystemSpecs;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
/// Cap each captured section so a pathological tool can't flood the report.
|
||||
const MAX_SECTION_BYTES: usize = 16 * 1024;
|
||||
|
||||
/// Run `cmd args…` and return its combined stdout/stderr, or a note that the
|
||||
/// tool is unavailable. Never fails: missing tools are part of the diagnosis.
|
||||
fn capture(cmd: &str, args: &[&str]) -> String {
|
||||
match std::process::Command::new(cmd).args(args).output() {
|
||||
Ok(out) => {
|
||||
let mut text = String::new();
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
if !stdout.trim().is_empty() {
|
||||
text.push_str(stdout.trim_end());
|
||||
}
|
||||
if !stderr.trim().is_empty() {
|
||||
if !text.is_empty() {
|
||||
text.push_str("\n--- stderr ---\n");
|
||||
}
|
||||
text.push_str(stderr.trim_end());
|
||||
}
|
||||
if text.is_empty() {
|
||||
text = format!("(no output, exit status: {})", out.status);
|
||||
}
|
||||
truncate(text)
|
||||
}
|
||||
Err(e) => format!("(not available: {e})"),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate(mut text: String) -> String {
|
||||
if text.len() > MAX_SECTION_BYTES {
|
||||
// Truncate on a char boundary at or below the cap.
|
||||
let mut cut = MAX_SECTION_BYTES;
|
||||
while !text.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
text.truncate(cut);
|
||||
text.push_str("\n… (truncated)");
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
fn section(report: &mut String, title: &str, body: &str) {
|
||||
let _ = writeln!(report, "## {title}\n```\n{body}\n```\n");
|
||||
}
|
||||
|
||||
/// Walk `/sys/class/drm/card*` and report the fields the sysfs detection
|
||||
/// paths read: vendor, device id, driver, and dedicated VRAM if exposed.
|
||||
fn sysfs_drm_summary() -> String {
|
||||
let entries = match std::fs::read_dir("/sys/class/drm") {
|
||||
Ok(e) => e,
|
||||
Err(e) => return format!("(not available: {e})"),
|
||||
};
|
||||
let mut cards: Vec<String> = Vec::new();
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|f| f.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if !name.starts_with("card") || name.contains('-') {
|
||||
continue;
|
||||
}
|
||||
let device = path.join("device");
|
||||
let read = |f: &str| {
|
||||
std::fs::read_to_string(device.join(f))
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_else(|_| "-".to_string())
|
||||
};
|
||||
let driver = std::fs::read_to_string(device.join("uevent"))
|
||||
.ok()
|
||||
.and_then(|u| {
|
||||
u.lines()
|
||||
.find(|l| l.starts_with("DRIVER="))
|
||||
.map(|l| l.trim_start_matches("DRIVER=").to_string())
|
||||
})
|
||||
.unwrap_or_else(|| "-".to_string());
|
||||
cards.push(format!(
|
||||
"{name}: vendor={} device={} driver={driver} mem_info_vram_total={}",
|
||||
read("vendor"),
|
||||
read("device"),
|
||||
read("mem_info_vram_total"),
|
||||
));
|
||||
}
|
||||
if cards.is_empty() {
|
||||
"(no /sys/class/drm cardN entries)".to_string()
|
||||
} else {
|
||||
cards.sort();
|
||||
cards.join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the full diagnostic report as Markdown.
|
||||
///
|
||||
/// `version` is the binary version string (core doesn't know the crate
|
||||
/// version of the caller).
|
||||
pub fn collect_diagnostics(version: &str) -> String {
|
||||
let mut report = String::new();
|
||||
let _ = writeln!(report, "# llmfit doctor report\n");
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"Paste this whole report into a GitHub issue at \
|
||||
https://github.com/AlexsJones/llmfit/issues — the raw tool output \
|
||||
below is what lets detection bugs become regression tests. It \
|
||||
contains hardware model names and driver info only.\n"
|
||||
);
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"- llmfit version: {version}\n- OS: {} ({})\n",
|
||||
std::env::consts::OS,
|
||||
std::env::consts::ARCH
|
||||
);
|
||||
|
||||
// What llmfit concluded — shown first so mismatches with the raw
|
||||
// output below are immediately visible.
|
||||
let specs = SystemSpecs::detect();
|
||||
section(&mut report, "Detected by llmfit", &format!("{specs:#?}"));
|
||||
|
||||
// NVIDIA
|
||||
section(
|
||||
&mut report,
|
||||
"nvidia-smi (extended query)",
|
||||
&capture(
|
||||
"nvidia-smi",
|
||||
&[
|
||||
"--query-gpu=addressing_mode,memory.total,name",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
),
|
||||
);
|
||||
section(
|
||||
&mut report,
|
||||
"nvidia-smi (standard query)",
|
||||
&capture(
|
||||
"nvidia-smi",
|
||||
&[
|
||||
"--query-gpu=memory.total,name",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
),
|
||||
);
|
||||
|
||||
// AMD ROCm
|
||||
section(
|
||||
&mut report,
|
||||
"rocm-smi --showmeminfo vram",
|
||||
&capture("rocm-smi", &["--showmeminfo", "vram"]),
|
||||
);
|
||||
section(
|
||||
&mut report,
|
||||
"rocm-smi --showproductname",
|
||||
&capture("rocm-smi", &["--showproductname"]),
|
||||
);
|
||||
|
||||
if cfg!(target_os = "linux") {
|
||||
section(&mut report, "sysfs DRM cards", &sysfs_drm_summary());
|
||||
section(&mut report, "lspci (display controllers)", &{
|
||||
let full = capture("lspci", &["-nn"]);
|
||||
let filtered: Vec<&str> = full
|
||||
.lines()
|
||||
.filter(|l| {
|
||||
let lower = l.to_lowercase();
|
||||
lower.contains("vga")
|
||||
|| lower.contains("3d controller")
|
||||
|| lower.contains("display controller")
|
||||
|| lower.starts_with("(not available")
|
||||
})
|
||||
.collect();
|
||||
if filtered.is_empty() {
|
||||
"(no display controllers listed)".to_string()
|
||||
} else {
|
||||
filtered.join("\n")
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if cfg!(target_os = "macos") {
|
||||
section(
|
||||
&mut report,
|
||||
"system_profiler SPDisplaysDataType",
|
||||
&capture("system_profiler", &["SPDisplaysDataType"]),
|
||||
);
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") {
|
||||
section(
|
||||
&mut report,
|
||||
"PowerShell Win32_VideoController",
|
||||
&capture(
|
||||
"powershell",
|
||||
&[
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
"Get-CimInstance Win32_VideoController | Select-Object Name,AdapterRAM | ForEach-Object { $_.Name + '|' + $_.AdapterRAM }",
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// Vulkan (fallback path on Linux/Windows) and NPUs — cheap to include
|
||||
// everywhere; reported as unavailable where the tool is missing.
|
||||
section(
|
||||
&mut report,
|
||||
"vulkaninfo --summary",
|
||||
&capture("vulkaninfo", &["--summary"]),
|
||||
);
|
||||
section(&mut report, "npu-smi info", &capture("npu-smi", &["info"]));
|
||||
|
||||
// Provider applications that are detected by install location rather
|
||||
// than a live API probe (#731) — lets "installed but not running"
|
||||
// reports carry the evidence.
|
||||
section(
|
||||
&mut report,
|
||||
"Provider app installs",
|
||||
&format!(
|
||||
"LM Studio installed: {}\nDocker Desktop installed: {}\nollama on PATH: {}",
|
||||
crate::providers::lmstudio_app_installed(),
|
||||
crate::providers::docker_desktop_installed(),
|
||||
crate::providers::command_exists("ollama"),
|
||||
),
|
||||
);
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_capture_missing_tool_is_note_not_panic() {
|
||||
let out = capture("definitely-not-a-real-tool-xyz", &[]);
|
||||
assert!(out.starts_with("(not available:"), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_caps_section_and_marks_it() {
|
||||
let big = "x".repeat(MAX_SECTION_BYTES + 100);
|
||||
let out = truncate(big);
|
||||
assert!(out.len() <= MAX_SECTION_BYTES + 20);
|
||||
assert!(out.ends_with("(truncated)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_report_contains_key_sections() {
|
||||
let report = collect_diagnostics("0.0.0-test");
|
||||
assert!(report.contains("# llmfit doctor report"));
|
||||
assert!(report.contains("llmfit version: 0.0.0-test"));
|
||||
assert!(report.contains("## Detected by llmfit"));
|
||||
assert!(report.contains("## rocm-smi --showmeminfo vram"));
|
||||
assert!(report.contains("## nvidia-smi (extended query)"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
pub mod analysis;
|
||||
pub mod bench;
|
||||
pub mod benchmarks;
|
||||
pub mod claim;
|
||||
pub mod doctor;
|
||||
pub mod fit;
|
||||
pub mod hardware;
|
||||
pub mod models;
|
||||
pub mod plan;
|
||||
pub mod providers;
|
||||
pub mod quality;
|
||||
pub mod share;
|
||||
pub mod task_bench;
|
||||
pub mod update;
|
||||
|
||||
pub use analysis::{InstalledIndex, build_model_fits};
|
||||
pub use fit::{FitLevel, InferenceRuntime, ModelFit, RunMode, ScoreComponents, SortColumn};
|
||||
pub use hardware::{GpuBackend, SystemSpecs};
|
||||
pub use models::{Capability, LlmModel, ModelDatabase, ModelFormat, UseCase};
|
||||
pub use plan::{
|
||||
HardwareEstimate, PathEstimate, PlanCurrentStatus, PlanEstimate, PlanRequest, PlanRunPath,
|
||||
UpgradeDelta, estimate_model_plan, normalize_quant, resolve_model_selector,
|
||||
};
|
||||
pub use providers::{
|
||||
LlamaCppProvider, LmStudioProvider, MlxProvider, ModelProvider, OllamaProvider, VllmProvider,
|
||||
};
|
||||
pub use update::{
|
||||
UpdateOptions, cache_file, clear_cache, load_cache, save_cache, update_model_cache,
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,934 @@
|
||||
//! Role-based quality benchmarking for model routing.
|
||||
//!
|
||||
//! Tests models against 13 Amplifier roles with scored rubrics,
|
||||
//! producing a routing matrix that maps roles to optimal models.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// HTTP response types are defined once in bench.rs and shared here to prevent
|
||||
// the two modules from silently diverging if fields are added or changed.
|
||||
use crate::bench::{ChatCompletionResponse, ChatUsage, OllamaGenResponse};
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────
|
||||
|
||||
/// A single scoring rule applied to a model response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ScoringRule {
|
||||
pub pattern: String,
|
||||
pub weight: i32,
|
||||
#[serde(default)]
|
||||
pub negate: bool,
|
||||
#[serde(default)]
|
||||
pub case_insensitive: bool,
|
||||
}
|
||||
|
||||
/// Definition of a single quality test loaded from YAML.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QualityTestDef {
|
||||
pub name: String,
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub rules: Vec<ScoringRule>,
|
||||
/// Weight given to speed in the composite score (default 1.0).
|
||||
#[serde(default = "default_speed_weight")]
|
||||
pub speed_weight: Option<f64>,
|
||||
/// Maximum tokens to generate (default 1024).
|
||||
#[serde(default)]
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Sampling temperature (default 0.3).
|
||||
#[serde(default)]
|
||||
pub temperature: Option<f64>,
|
||||
}
|
||||
|
||||
fn default_speed_weight() -> Option<f64> {
|
||||
Some(1.0)
|
||||
}
|
||||
|
||||
/// A role definition containing its description and test suite.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RoleDef {
|
||||
pub description: String,
|
||||
pub tests: Vec<QualityTestDef>,
|
||||
}
|
||||
|
||||
/// Top-level quality benchmark configuration.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QualityConfig {
|
||||
pub roles: BTreeMap<String, RoleDef>,
|
||||
}
|
||||
|
||||
/// Result of a single quality test against one model.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QualityResult {
|
||||
pub test_name: String,
|
||||
pub role: String,
|
||||
/// Quality score from rubric evaluation (0-10).
|
||||
pub quality: f64,
|
||||
/// Output tokens per second.
|
||||
pub tok_per_sec: f64,
|
||||
/// Composite score blending quality and speed.
|
||||
pub composite: f64,
|
||||
/// First 150 chars of the model response.
|
||||
pub response_preview: String,
|
||||
/// Time to first token in milliseconds (if available).
|
||||
pub ttft_ms: Option<f64>,
|
||||
/// Total wall-clock time in seconds.
|
||||
pub wall_time_sec: f64,
|
||||
/// Number of output tokens.
|
||||
pub eval_tokens: u64,
|
||||
/// Error message if the test failed.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Aggregated score for a single role across all its tests.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RoleScore {
|
||||
pub role: String,
|
||||
pub quality: f64,
|
||||
pub speed: f64,
|
||||
pub composite: f64,
|
||||
pub test_count: usize,
|
||||
}
|
||||
|
||||
/// Complete quality results for one model.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelQualityResult {
|
||||
pub model: String,
|
||||
pub provider: String,
|
||||
pub roles: Vec<RoleScore>,
|
||||
pub test_results: Vec<QualityResult>,
|
||||
pub overall_quality: f64,
|
||||
pub overall_speed: f64,
|
||||
pub overall_composite: f64,
|
||||
}
|
||||
|
||||
/// Routing recommendation: which model is best for a given role.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RoutingRecommendation {
|
||||
pub role: String,
|
||||
pub model: String,
|
||||
pub quality: f64,
|
||||
pub speed: f64,
|
||||
pub composite: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
// ── Scoring ────────────────────────────────────────────────────────
|
||||
|
||||
/// Evaluate a model response against a set of scoring rules.
|
||||
///
|
||||
/// Returns a quality score clamped to 0-10.
|
||||
pub fn evaluate_response(text: &str, rules: &[ScoringRule]) -> f64 {
|
||||
let mut score: i32 = 0;
|
||||
for rule in rules {
|
||||
let flags = if rule.case_insensitive { "(?i)" } else { "" };
|
||||
let pattern = format!("{}{}", flags, rule.pattern);
|
||||
let matched = Regex::new(&pattern)
|
||||
.map(|re| re.is_match(text))
|
||||
.unwrap_or(false);
|
||||
|
||||
if rule.negate {
|
||||
if !matched {
|
||||
score += rule.weight;
|
||||
}
|
||||
} else if matched {
|
||||
score += rule.weight;
|
||||
}
|
||||
}
|
||||
(score.max(0).min(10)) as f64
|
||||
}
|
||||
|
||||
/// Extract a code block from a markdown-fenced response.
|
||||
///
|
||||
/// Returns the content inside the first ``` block, or the full text
|
||||
/// if no fences are found.
|
||||
pub fn extract_code_block(text: &str) -> String {
|
||||
if let Some(start) = text.find("```") {
|
||||
// Skip past the opening fence + optional language tag
|
||||
let after_fence = &text[start + 3..];
|
||||
let content_start = after_fence.find('\n').map(|i| i + 1).unwrap_or(0);
|
||||
let content = &after_fence[content_start..];
|
||||
if let Some(end) = content.find("```") {
|
||||
return content[..end].trim().to_string();
|
||||
}
|
||||
return content.trim().to_string();
|
||||
}
|
||||
text.trim().to_string()
|
||||
}
|
||||
|
||||
// ── HTTP helpers (following bench.rs ureq patterns) ────────────────
|
||||
|
||||
/// Raw response from an inference call, carrying text + timing info.
|
||||
pub struct InferenceResponse {
|
||||
pub text: String,
|
||||
pub eval_count: u64,
|
||||
pub tok_per_sec: f64,
|
||||
pub ttft_ms: Option<f64>,
|
||||
pub wall_time_sec: f64,
|
||||
}
|
||||
|
||||
pub fn quality_ollama_generate(
|
||||
url: &str,
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
max_tokens: u32,
|
||||
temperature: f64,
|
||||
) -> Result<InferenceResponse, String> {
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = ureq::post(url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(600)))
|
||||
.build()
|
||||
.send_json(&body)
|
||||
.map_err(|e| format!("Ollama request failed: {}", e))?;
|
||||
|
||||
let wall_time = start.elapsed();
|
||||
|
||||
let resp_body: OllamaGenResponse = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("Ollama JSON parse error: {}", e))?;
|
||||
|
||||
let eval_count = resp_body.eval_count.unwrap_or(0);
|
||||
|
||||
let ttft_ms = resp_body
|
||||
.prompt_eval_duration
|
||||
.map(|ns| ns as f64 / 1_000_000.0);
|
||||
|
||||
let tok_per_sec = if let (Some(ec), Some(ed)) = (resp_body.eval_count, resp_body.eval_duration)
|
||||
{
|
||||
if ed > 0 {
|
||||
ec as f64 / (ed as f64 / 1_000_000_000.0)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
} else if eval_count > 0 {
|
||||
eval_count as f64 / wall_time.as_secs_f64()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(InferenceResponse {
|
||||
text: resp_body.response,
|
||||
eval_count,
|
||||
tok_per_sec,
|
||||
ttft_ms,
|
||||
wall_time_sec: wall_time.as_secs_f64(),
|
||||
})
|
||||
}
|
||||
|
||||
fn quality_openai_chat(
|
||||
url: &str,
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
max_tokens: u32,
|
||||
temperature: f64,
|
||||
) -> Result<InferenceResponse, String> {
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": false,
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = ureq::post(url)
|
||||
.config()
|
||||
.timeout_global(Some(Duration::from_secs(600)))
|
||||
.build()
|
||||
.send_json(&body)
|
||||
.map_err(|e| format!("OpenAI-compat request failed: {}", e))?;
|
||||
|
||||
let wall_time = start.elapsed();
|
||||
|
||||
let completion: ChatCompletionResponse = resp
|
||||
.into_body()
|
||||
.read_json()
|
||||
.map_err(|e| format!("JSON parse error: {}", e))?;
|
||||
|
||||
let usage = completion.usage.unwrap_or(ChatUsage {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
});
|
||||
|
||||
let text = completion
|
||||
.choices
|
||||
.first()
|
||||
.and_then(|c| c.message.as_ref())
|
||||
.and_then(|m| m.content.clone())
|
||||
.unwrap_or_default();
|
||||
|
||||
let eval_count = usage.completion_tokens as u64;
|
||||
let tok_per_sec = if eval_count > 0 && wall_time.as_secs_f64() > 0.0 {
|
||||
eval_count as f64 / wall_time.as_secs_f64()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(InferenceResponse {
|
||||
text,
|
||||
eval_count,
|
||||
tok_per_sec,
|
||||
ttft_ms: None,
|
||||
wall_time_sec: wall_time.as_secs_f64(),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Public benchmark entry points ──────────────────────────────────
|
||||
|
||||
/// Run quality benchmarks against an Ollama endpoint.
|
||||
///
|
||||
/// Iterates through all roles/tests in `config`, sends prompts to the
|
||||
/// model, scores responses, and returns aggregated results.
|
||||
pub fn bench_quality_ollama(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
config: &QualityConfig,
|
||||
role_filter: Option<&[String]>,
|
||||
) -> Result<ModelQualityResult, String> {
|
||||
let url = format!("{}/api/generate", base_url.trim_end_matches('/'));
|
||||
|
||||
// Warmup
|
||||
let _ = quality_ollama_generate(&url, model, "Say hello.", 64, 0.3);
|
||||
|
||||
run_all_tests(
|
||||
model,
|
||||
"ollama",
|
||||
config,
|
||||
role_filter,
|
||||
|prompt, max_tok, temp| quality_ollama_generate(&url, model, prompt, max_tok, temp),
|
||||
)
|
||||
}
|
||||
|
||||
/// Run quality benchmarks against an OpenAI-compatible endpoint (vLLM, MLX).
|
||||
pub fn bench_quality_openai_compat(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
provider: &str,
|
||||
config: &QualityConfig,
|
||||
role_filter: Option<&[String]>,
|
||||
) -> Result<ModelQualityResult, String> {
|
||||
let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
|
||||
|
||||
// Warmup
|
||||
let _ = quality_openai_chat(&url, model, "Say hello.", 64, 0.3);
|
||||
|
||||
run_all_tests(
|
||||
model,
|
||||
provider,
|
||||
config,
|
||||
role_filter,
|
||||
|prompt, max_tok, temp| quality_openai_chat(&url, model, prompt, max_tok, temp),
|
||||
)
|
||||
}
|
||||
|
||||
/// Shared test runner used by both Ollama and OpenAI-compat paths.
|
||||
fn run_all_tests<F>(
|
||||
model: &str,
|
||||
provider: &str,
|
||||
config: &QualityConfig,
|
||||
role_filter: Option<&[String]>,
|
||||
generate: F,
|
||||
) -> Result<ModelQualityResult, String>
|
||||
where
|
||||
F: Fn(&str, u32, f64) -> Result<InferenceResponse, String>,
|
||||
{
|
||||
let mut all_results = Vec::new();
|
||||
let mut role_scores = Vec::new();
|
||||
|
||||
for (role_name, role_def) in &config.roles {
|
||||
// Skip roles not in the filter (if a filter is set)
|
||||
if let Some(filter) = role_filter
|
||||
&& !filter.iter().any(|f| f == role_name)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut role_results = Vec::new();
|
||||
|
||||
for test_def in &role_def.tests {
|
||||
let max_tokens = test_def.max_tokens.unwrap_or(1024);
|
||||
let temperature = test_def.temperature.unwrap_or(0.3);
|
||||
|
||||
let result = match generate(&test_def.prompt, max_tokens, temperature) {
|
||||
Ok(resp) => {
|
||||
let quality = evaluate_response(&resp.text, &test_def.rules);
|
||||
let speed_weight = test_def.speed_weight.unwrap_or(1.0);
|
||||
let speed_norm = (resp.tok_per_sec / 3.0).min(10.0);
|
||||
let composite =
|
||||
(quality * 2.0 + speed_norm * speed_weight) / (2.0 + speed_weight);
|
||||
|
||||
QualityResult {
|
||||
test_name: test_def.name.clone(),
|
||||
role: role_name.clone(),
|
||||
quality,
|
||||
tok_per_sec: resp.tok_per_sec,
|
||||
composite,
|
||||
response_preview: resp.text.chars().take(150).collect(),
|
||||
ttft_ms: resp.ttft_ms,
|
||||
wall_time_sec: resp.wall_time_sec,
|
||||
eval_tokens: resp.eval_count,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
Err(e) => QualityResult {
|
||||
test_name: test_def.name.clone(),
|
||||
role: role_name.clone(),
|
||||
quality: 0.0,
|
||||
tok_per_sec: 0.0,
|
||||
composite: 0.0,
|
||||
response_preview: String::new(),
|
||||
ttft_ms: None,
|
||||
wall_time_sec: 0.0,
|
||||
eval_tokens: 0,
|
||||
error: Some(e),
|
||||
},
|
||||
};
|
||||
|
||||
role_results.push(result.clone());
|
||||
all_results.push(result);
|
||||
}
|
||||
|
||||
// Compute role averages
|
||||
let n = role_results.len() as f64;
|
||||
if n > 0.0 {
|
||||
let avg_q = role_results.iter().map(|r| r.quality).sum::<f64>() / n;
|
||||
let avg_s = role_results.iter().map(|r| r.tok_per_sec).sum::<f64>() / n;
|
||||
let avg_c = role_results.iter().map(|r| r.composite).sum::<f64>() / n;
|
||||
|
||||
role_scores.push(RoleScore {
|
||||
role: role_name.clone(),
|
||||
quality: (avg_q * 10.0).round() / 10.0,
|
||||
speed: (avg_s * 10.0).round() / 10.0,
|
||||
composite: (avg_c * 10.0).round() / 10.0,
|
||||
test_count: role_results.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Compute overall averages
|
||||
let n_roles = role_scores.len() as f64;
|
||||
let overall_quality = if n_roles > 0.0 {
|
||||
role_scores.iter().map(|r| r.quality).sum::<f64>() / n_roles
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let overall_speed = if n_roles > 0.0 {
|
||||
role_scores.iter().map(|r| r.speed).sum::<f64>() / n_roles
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let overall_composite = if n_roles > 0.0 {
|
||||
role_scores.iter().map(|r| r.composite).sum::<f64>() / n_roles
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(ModelQualityResult {
|
||||
model: model.to_string(),
|
||||
provider: provider.to_string(),
|
||||
roles: role_scores,
|
||||
test_results: all_results,
|
||||
overall_quality,
|
||||
overall_speed,
|
||||
overall_composite,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Routing matrix ─────────────────────────────────────────────────
|
||||
|
||||
/// Compute the best model for each role based on composite scores.
|
||||
pub fn compute_routing(results: &[ModelQualityResult]) -> Vec<RoutingRecommendation> {
|
||||
let mut role_map: HashMap<String, Vec<(String, f64, f64, f64)>> = HashMap::new();
|
||||
|
||||
for mr in results {
|
||||
for rs in &mr.roles {
|
||||
role_map.entry(rs.role.clone()).or_default().push((
|
||||
mr.model.clone(),
|
||||
rs.quality,
|
||||
rs.speed,
|
||||
rs.composite,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut routing = Vec::new();
|
||||
for (role, mut scores) in role_map {
|
||||
scores.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap_or(std::cmp::Ordering::Equal));
|
||||
if let Some((model, q, s, c)) = scores.first() {
|
||||
routing.push(RoutingRecommendation {
|
||||
role,
|
||||
model: model.clone(),
|
||||
quality: *q,
|
||||
speed: *s,
|
||||
composite: *c,
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
routing.sort_by(|a, b| a.role.cmp(&b.role));
|
||||
routing
|
||||
}
|
||||
|
||||
/// Compute runner-up models for each role with contextual notes.
|
||||
pub fn compute_runner_ups(results: &[ModelQualityResult]) -> Vec<RoutingRecommendation> {
|
||||
let mut role_map: HashMap<String, Vec<(String, f64, f64, f64)>> = HashMap::new();
|
||||
|
||||
for mr in results {
|
||||
for rs in &mr.roles {
|
||||
role_map.entry(rs.role.clone()).or_default().push((
|
||||
mr.model.clone(),
|
||||
rs.quality,
|
||||
rs.speed,
|
||||
rs.composite,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut runner_ups = Vec::new();
|
||||
for (role, mut scores) in role_map {
|
||||
scores.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
if scores.len() >= 2 {
|
||||
let (ref _best_model, best_q, best_s, best_c) = scores[0];
|
||||
let (ref ru_model, ru_q, ru_s, ru_c) = scores[1];
|
||||
|
||||
let note = if ru_s > best_s * 1.5 {
|
||||
"consider for speed".to_string()
|
||||
} else if (ru_q - best_q).abs() < 1.0 {
|
||||
"close quality".to_string()
|
||||
} else if ru_c > best_c * 0.9 {
|
||||
"competitive composite".to_string()
|
||||
} else {
|
||||
"alternative".to_string()
|
||||
};
|
||||
|
||||
runner_ups.push(RoutingRecommendation {
|
||||
role,
|
||||
model: ru_model.clone(),
|
||||
quality: ru_q,
|
||||
speed: ru_s,
|
||||
composite: ru_c,
|
||||
note: Some(note),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
runner_ups.sort_by(|a, b| a.role.cmp(&b.role));
|
||||
runner_ups
|
||||
}
|
||||
|
||||
// ── YAML config loading ────────────────────────────────────────────
|
||||
|
||||
/// Parse a YAML string into a `QualityConfig`.
|
||||
pub fn load_quality_config(yaml: &str) -> Result<QualityConfig, String> {
|
||||
serde_yml::from_str(yaml).map_err(|e| format!("Failed to parse quality config: {}", e))
|
||||
}
|
||||
|
||||
/// Return the built-in default quality config (embedded from `data/benchmarks.yaml`).
|
||||
pub fn default_quality_config() -> QualityConfig {
|
||||
let yaml = include_str!("../data/benchmarks.yaml");
|
||||
load_quality_config(yaml).expect("embedded benchmarks.yaml is invalid")
|
||||
}
|
||||
|
||||
// ── Display helpers ────────────────────────────────────────────────
|
||||
|
||||
impl ModelQualityResult {
|
||||
/// Print a human-readable summary of quality results.
|
||||
pub fn display(&self) {
|
||||
println!();
|
||||
println!(" === Quality Benchmark Results ===");
|
||||
println!(" Model: {}", self.model);
|
||||
println!(" Provider: {}", self.provider);
|
||||
println!();
|
||||
println!(
|
||||
" Overall: quality={:.1} speed={:.1} tok/s composite={:.1}",
|
||||
self.overall_quality, self.overall_speed, self.overall_composite
|
||||
);
|
||||
println!();
|
||||
println!(" Role Quality Speed Composite Tests");
|
||||
println!(" ─────────────── ─────── ─────── ───────── ─────");
|
||||
for rs in &self.roles {
|
||||
println!(
|
||||
" {:<15} {:>6.1} {:>6.1} {:>8.1} {:>4}",
|
||||
rs.role, rs.quality, rs.speed, rs.composite, rs.test_count
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
/// Print results as JSON.
|
||||
pub fn display_json(&self) {
|
||||
let json = serde_json::json!({
|
||||
"quality_benchmark": {
|
||||
"model": self.model,
|
||||
"provider": self.provider,
|
||||
"overall": {
|
||||
"quality": self.overall_quality,
|
||||
"speed": self.overall_speed,
|
||||
"composite": self.overall_composite,
|
||||
},
|
||||
"role_scores": self.roles,
|
||||
"test_results": self.test_results,
|
||||
}
|
||||
});
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json).expect("JSON serialization failed")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl RoutingRecommendation {
|
||||
/// Print a routing matrix row.
|
||||
pub fn display_row(&self) {
|
||||
let note_str = self
|
||||
.note
|
||||
.as_deref()
|
||||
.map(|n| format!(" ({})", n))
|
||||
.unwrap_or_default();
|
||||
println!(
|
||||
" {:<17} -> {:<30} q={:.1} s={:.1} c={:.1}{}",
|
||||
self.role, self.model, self.quality, self.speed, self.composite, note_str
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Print a full routing matrix.
|
||||
pub fn display_routing_matrix(
|
||||
routing: &[RoutingRecommendation],
|
||||
runner_ups: &[RoutingRecommendation],
|
||||
) {
|
||||
println!();
|
||||
println!(" === Routing Matrix ===");
|
||||
println!(" Best model per role:");
|
||||
println!();
|
||||
for r in routing {
|
||||
r.display_row();
|
||||
}
|
||||
|
||||
if !runner_ups.is_empty() {
|
||||
println!();
|
||||
println!(" Runner-ups:");
|
||||
println!();
|
||||
for r in runner_ups {
|
||||
r.display_row();
|
||||
}
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// ── Frontier model baselines ──────────────────────────────────────
|
||||
|
||||
/// A baseline score for a frontier model on a specific role.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BaselineScore {
|
||||
pub quality: f64,
|
||||
pub speed: f64,
|
||||
pub composite: f64,
|
||||
}
|
||||
|
||||
/// Baseline results for a frontier model.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BaselineModel {
|
||||
pub model: String,
|
||||
pub provider: String,
|
||||
pub roles: HashMap<String, BaselineScore>,
|
||||
pub overall: BaselineScore,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct BaselinesFile {
|
||||
baselines: Vec<BaselineModel>,
|
||||
}
|
||||
|
||||
/// Load embedded frontier model baselines.
|
||||
pub fn load_baselines() -> Vec<BaselineModel> {
|
||||
let json = include_str!("../data/baselines.json");
|
||||
serde_json::from_str::<BaselinesFile>(json)
|
||||
.map(|f| f.baselines)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Compare a local model's role scores against frontier baselines.
|
||||
/// Returns (role, local_composite, baseline_name, baseline_composite, pct_of_frontier).
|
||||
pub fn compare_to_baselines(
|
||||
result: &ModelQualityResult,
|
||||
baselines: &[BaselineModel],
|
||||
) -> Vec<(String, f64, String, f64, f64)> {
|
||||
let mut comparisons = Vec::new();
|
||||
for rs in &result.roles {
|
||||
// Find best baseline for this role
|
||||
let best_baseline = baselines
|
||||
.iter()
|
||||
.filter_map(|b| {
|
||||
b.roles
|
||||
.get(&rs.role)
|
||||
.map(|bs| (b.model.as_str(), bs.composite))
|
||||
})
|
||||
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
if let Some((baseline_model, baseline_composite)) = best_baseline {
|
||||
let pct = if baseline_composite > 0.0 {
|
||||
(rs.composite / baseline_composite) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
comparisons.push((
|
||||
rs.role.clone(),
|
||||
rs.composite,
|
||||
baseline_model.to_string(),
|
||||
baseline_composite,
|
||||
pct,
|
||||
));
|
||||
}
|
||||
}
|
||||
comparisons
|
||||
}
|
||||
|
||||
// ── Tests ──────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_response_basic_match() {
|
||||
let rules = vec![
|
||||
ScoringRule {
|
||||
pattern: "hello".to_string(),
|
||||
weight: 3,
|
||||
negate: false,
|
||||
case_insensitive: true,
|
||||
},
|
||||
ScoringRule {
|
||||
pattern: "world".to_string(),
|
||||
weight: 2,
|
||||
negate: false,
|
||||
case_insensitive: false,
|
||||
},
|
||||
];
|
||||
assert!((evaluate_response("Hello world", &rules) - 5.0).abs() < 0.01);
|
||||
assert!((evaluate_response("Hello World", &rules) - 3.0).abs() < 0.01);
|
||||
assert!((evaluate_response("goodbye", &rules) - 0.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_response_negate() {
|
||||
let rules = vec![ScoringRule {
|
||||
pattern: "error".to_string(),
|
||||
weight: 5,
|
||||
negate: true,
|
||||
case_insensitive: false,
|
||||
}];
|
||||
// negate: score if pattern NOT found
|
||||
assert!((evaluate_response("all good", &rules) - 5.0).abs() < 0.01);
|
||||
assert!((evaluate_response("found error", &rules) - 0.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_response_clamp() {
|
||||
let rules = vec![
|
||||
ScoringRule {
|
||||
pattern: "a".to_string(),
|
||||
weight: 8,
|
||||
negate: false,
|
||||
case_insensitive: false,
|
||||
},
|
||||
ScoringRule {
|
||||
pattern: "b".to_string(),
|
||||
weight: 8,
|
||||
negate: false,
|
||||
case_insensitive: false,
|
||||
},
|
||||
];
|
||||
// Both match = 16, but clamped to 10
|
||||
assert!((evaluate_response("a b", &rules) - 10.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_response_negative_clamp() {
|
||||
let rules = vec![ScoringRule {
|
||||
pattern: "bad".to_string(),
|
||||
weight: -5,
|
||||
negate: false,
|
||||
case_insensitive: false,
|
||||
}];
|
||||
// Negative score clamped to 0
|
||||
assert!((evaluate_response("bad", &rules) - 0.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_code_block() {
|
||||
let md = "Here is the code:\n```python\ndef hello():\n print('hi')\n```\nDone.";
|
||||
assert_eq!(extract_code_block(md), "def hello():\n print('hi')");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_code_block_no_fence() {
|
||||
let plain = "def hello():\n print('hi')";
|
||||
assert_eq!(extract_code_block(plain), plain.trim());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_quality_config() {
|
||||
let yaml = r#"
|
||||
roles:
|
||||
general:
|
||||
description: General tasks
|
||||
tests:
|
||||
- name: test1
|
||||
prompt: "Say hello"
|
||||
rules:
|
||||
- { pattern: "hello", weight: 5, case_insensitive: true }
|
||||
"#;
|
||||
let config = load_quality_config(yaml).unwrap();
|
||||
assert!(config.roles.contains_key("general"));
|
||||
assert_eq!(config.roles["general"].tests.len(), 1);
|
||||
assert_eq!(config.roles["general"].tests[0].rules.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_quality_config_loads() {
|
||||
let config = default_quality_config();
|
||||
assert!(
|
||||
!config.roles.is_empty(),
|
||||
"default config should have at least one role"
|
||||
);
|
||||
// Check a known role exists
|
||||
assert!(
|
||||
config.roles.contains_key("general"),
|
||||
"default config should have 'general' role"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_routing_single_model() {
|
||||
let results = vec![ModelQualityResult {
|
||||
model: "test-model".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
roles: vec![
|
||||
RoleScore {
|
||||
role: "general".to_string(),
|
||||
quality: 7.0,
|
||||
speed: 30.0,
|
||||
composite: 6.5,
|
||||
test_count: 3,
|
||||
},
|
||||
RoleScore {
|
||||
role: "coding".to_string(),
|
||||
quality: 8.0,
|
||||
speed: 25.0,
|
||||
composite: 7.0,
|
||||
test_count: 5,
|
||||
},
|
||||
],
|
||||
test_results: vec![],
|
||||
overall_quality: 7.5,
|
||||
overall_speed: 27.5,
|
||||
overall_composite: 6.75,
|
||||
}];
|
||||
|
||||
let routing = compute_routing(&results);
|
||||
assert_eq!(routing.len(), 2);
|
||||
assert!(routing.iter().all(|r| r.model == "test-model"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_routing_picks_best() {
|
||||
let results = vec![
|
||||
ModelQualityResult {
|
||||
model: "fast-model".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
roles: vec![RoleScore {
|
||||
role: "fast".to_string(),
|
||||
quality: 5.0,
|
||||
speed: 100.0,
|
||||
composite: 8.0,
|
||||
test_count: 2,
|
||||
}],
|
||||
test_results: vec![],
|
||||
overall_quality: 5.0,
|
||||
overall_speed: 100.0,
|
||||
overall_composite: 8.0,
|
||||
},
|
||||
ModelQualityResult {
|
||||
model: "smart-model".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
roles: vec![RoleScore {
|
||||
role: "fast".to_string(),
|
||||
quality: 9.0,
|
||||
speed: 10.0,
|
||||
composite: 7.0,
|
||||
test_count: 2,
|
||||
}],
|
||||
test_results: vec![],
|
||||
overall_quality: 9.0,
|
||||
overall_speed: 10.0,
|
||||
overall_composite: 7.0,
|
||||
},
|
||||
];
|
||||
|
||||
let routing = compute_routing(&results);
|
||||
assert_eq!(routing.len(), 1);
|
||||
assert_eq!(routing[0].model, "fast-model"); // higher composite
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_runner_ups() {
|
||||
let results = vec![
|
||||
ModelQualityResult {
|
||||
model: "best".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
roles: vec![RoleScore {
|
||||
role: "coding".to_string(),
|
||||
quality: 9.0,
|
||||
speed: 20.0,
|
||||
composite: 8.0,
|
||||
test_count: 3,
|
||||
}],
|
||||
test_results: vec![],
|
||||
overall_quality: 9.0,
|
||||
overall_speed: 20.0,
|
||||
overall_composite: 8.0,
|
||||
},
|
||||
ModelQualityResult {
|
||||
model: "second".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
roles: vec![RoleScore {
|
||||
role: "coding".to_string(),
|
||||
quality: 7.0,
|
||||
speed: 50.0,
|
||||
composite: 6.0,
|
||||
test_count: 3,
|
||||
}],
|
||||
test_results: vec![],
|
||||
overall_quality: 7.0,
|
||||
overall_speed: 50.0,
|
||||
overall_composite: 6.0,
|
||||
},
|
||||
];
|
||||
|
||||
let runner_ups = compute_runner_ups(&results);
|
||||
assert_eq!(runner_ups.len(), 1);
|
||||
assert_eq!(runner_ups[0].model, "second");
|
||||
assert!(runner_ups[0].note.is_some());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,90 @@
|
||||
//! Curated per-family use-case benchmark scores (issue #150).
|
||||
//!
|
||||
//! Maps model families to relative task-strength scores (0-100) aggregated
|
||||
//! from public leaderboards, so a strong coding model can outrank a larger
|
||||
//! generalist when the user asks for a coding recommendation. The table is
|
||||
//! embedded from `data/use_case_benchmarks.json` and refreshed alongside the
|
||||
//! weekly model-database update; [`score`] returns `None` for models without
|
||||
//! an entry, which fall back to the name heuristics in `fit.rs`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const TASK_BENCH_JSON: &str = include_str!("../data/use_case_benchmarks.json");
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct FamilyEntry {
|
||||
#[serde(rename = "match")]
|
||||
patterns: Vec<String>,
|
||||
scores: HashMap<String, f64>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct BenchFile {
|
||||
families: Vec<FamilyEntry>,
|
||||
}
|
||||
|
||||
fn table() -> &'static [FamilyEntry] {
|
||||
static TABLE: OnceLock<Vec<FamilyEntry>> = OnceLock::new();
|
||||
TABLE.get_or_init(|| {
|
||||
serde_json::from_str::<BenchFile>(TASK_BENCH_JSON)
|
||||
.expect("embedded use_case_benchmarks.json is invalid")
|
||||
.families
|
||||
})
|
||||
}
|
||||
|
||||
/// Benchmark score for a model on a task (`"coding"`, `"reasoning"`,
|
||||
/// `"chat"`), or `None` if no family entry matches.
|
||||
///
|
||||
/// `name_lower` must already be lowercased. When several patterns match
|
||||
/// (e.g. `qwen3` and `qwen3-coder`), the longest — most specific — wins.
|
||||
pub fn score(name_lower: &str, task: &str) -> Option<f64> {
|
||||
let mut best: Option<(usize, f64)> = None;
|
||||
for entry in table() {
|
||||
for pattern in &entry.patterns {
|
||||
if name_lower.contains(pattern.as_str())
|
||||
&& let Some(s) = entry.scores.get(task)
|
||||
&& best.is_none_or(|(len, _)| pattern.len() > len)
|
||||
{
|
||||
best = Some((pattern.len(), *s));
|
||||
}
|
||||
}
|
||||
}
|
||||
best.map(|(_, s)| s)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_embedded_table_parses() {
|
||||
assert!(!table().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_longest_pattern_wins() {
|
||||
// "qwen3.5-coder-…" matches both "qwen3.5" and "qwen3.5-coder";
|
||||
// the coder entry must win for the coding task.
|
||||
let coder = score("qwen/qwen3.5-coder-32b-instruct", "coding").unwrap();
|
||||
let base = score("qwen/qwen3.5-32b-instruct", "coding").unwrap();
|
||||
assert!(coder > base, "coder {coder} <= base {base}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_family_is_none() {
|
||||
assert_eq!(score("acme/customnet-7b", "coding"), None);
|
||||
assert_eq!(score("qwen/qwen3.5-32b", "nonexistent-task"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_coding_specialist_beats_generalist_at_coding_only() {
|
||||
let starcoder_code = score("bigcode/starcoder2-15b", "coding").unwrap();
|
||||
let llama_code = score("meta-llama/llama-3.3-70b", "coding").unwrap();
|
||||
assert!(starcoder_code > llama_code);
|
||||
|
||||
let starcoder_chat = score("bigcode/starcoder2-15b", "chat").unwrap();
|
||||
let llama_chat = score("meta-llama/llama-3.3-70b", "chat").unwrap();
|
||||
assert!(llama_chat > starcoder_chat);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
use llmfit_core::{ModelDatabase, ModelFormat};
|
||||
|
||||
const CORE_ONNX_MODELS_JSON: &str = include_str!("../data/onnx_models.json");
|
||||
const ROOT_ONNX_MODELS_JSON: &str = include_str!("../../data/onnx_models.json");
|
||||
|
||||
fn load_catalog() -> Vec<serde_json::Value> {
|
||||
let value: serde_json::Value = serde_json::from_str(CORE_ONNX_MODELS_JSON)
|
||||
.expect("embedded onnx_models.json is valid JSON");
|
||||
value
|
||||
.as_array()
|
||||
.expect("embedded onnx_models.json is a JSON array")
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_and_repo_root_onnx_catalogs_are_mirrored() {
|
||||
assert_eq!(
|
||||
CORE_ONNX_MODELS_JSON, ROOT_ONNX_MODELS_JSON,
|
||||
"repo-root and llmfit-core ONNX catalogs must stay in sync"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn onnx_catalog_is_non_empty() {
|
||||
let models = load_catalog();
|
||||
assert!(
|
||||
!models.is_empty(),
|
||||
"embedded onnx_models.json must contain at least one model"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_model_is_marked_as_onnx() {
|
||||
for model in load_catalog() {
|
||||
assert_eq!(
|
||||
model.get("format").and_then(|v| v.as_str()),
|
||||
Some("onnx"),
|
||||
"model {:?} must set format = \"onnx\"",
|
||||
model.get("id")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_model_has_positive_quantization_sizes() {
|
||||
for model in load_catalog() {
|
||||
let onnx_files = model
|
||||
.get("onnx_files")
|
||||
.and_then(|v| v.as_object())
|
||||
.unwrap_or_else(|| panic!("model {:?} must include onnx_files", model.get("id")));
|
||||
assert!(
|
||||
!onnx_files.is_empty(),
|
||||
"model {:?} must list at least one quantization",
|
||||
model.get("id")
|
||||
);
|
||||
for (quant, size) in onnx_files {
|
||||
let bytes = size.as_u64().unwrap_or(0);
|
||||
assert!(
|
||||
bytes > 0,
|
||||
"quantization {} for model {:?} must have a positive byte size",
|
||||
quant,
|
||||
model.get("id")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_ids_are_unique() {
|
||||
let models = load_catalog();
|
||||
let mut ids: Vec<&str> = models
|
||||
.iter()
|
||||
.filter_map(|m| m.get("id").and_then(|v| v.as_str()))
|
||||
.collect();
|
||||
assert_eq!(ids.len(), models.len(), "every model must have a string id");
|
||||
let total = ids.len();
|
||||
ids.sort_unstable();
|
||||
ids.dedup();
|
||||
assert_eq!(ids.len(), total, "model ids must be unique");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_database_contains_onnx_catalog_models() {
|
||||
let catalog = load_catalog();
|
||||
let db = ModelDatabase::embedded();
|
||||
|
||||
for entry in catalog {
|
||||
let id = entry
|
||||
.get("id")
|
||||
.and_then(|v| v.as_str())
|
||||
.expect("catalog entry has id");
|
||||
let model = db
|
||||
.get_all_models()
|
||||
.iter()
|
||||
.find(|model| model.name == id)
|
||||
.unwrap_or_else(|| panic!("embedded database must include ONNX model {id}"));
|
||||
|
||||
assert_eq!(model.format, ModelFormat::Onnx);
|
||||
assert_eq!(model.provider, id.split('/').next().unwrap_or_default());
|
||||
assert!(
|
||||
model.min_ram_gb > 0.0,
|
||||
"ONNX model {id} must have a positive RAM estimate"
|
||||
);
|
||||
assert!(
|
||||
model.recommended_ram_gb >= model.min_ram_gb,
|
||||
"ONNX model {id} recommended RAM must be at least min RAM"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use jsonschema::Validator;
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
|
||||
fn load_schema() -> Value {
|
||||
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/schema.json");
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display()));
|
||||
serde_json::from_str(&text).unwrap_or_else(|e| panic!("cannot parse schema JSON: {e}"))
|
||||
}
|
||||
|
||||
fn validate_file(schema: &Validator, rel_path: &str) {
|
||||
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(rel_path);
|
||||
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display()));
|
||||
let data: Value = serde_json::from_str(&text)
|
||||
.unwrap_or_else(|e| panic!("cannot parse {}: {e}", path.display()));
|
||||
|
||||
let errors: Vec<String> = schema
|
||||
.iter_errors(&data)
|
||||
.take(30)
|
||||
.map(|e| format!(" [{}] {}", e.instance_path(), e))
|
||||
.collect();
|
||||
|
||||
assert!(
|
||||
errors.is_empty(),
|
||||
"{rel_path}: {} schema violation(s):\n{}",
|
||||
errors.len(),
|
||||
errors.join("\n")
|
||||
);
|
||||
|
||||
let count = data.as_array().map(|a| a.len()).unwrap_or(0);
|
||||
println!("✓ {rel_path} ({count} models)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_models_match_schema() {
|
||||
let schema_value = load_schema();
|
||||
let schema = jsonschema::validator_for(&schema_value)
|
||||
.expect("schema itself is invalid — check llmfit-core/data/schema.json");
|
||||
|
||||
validate_file(&schema, "data/hf_models.json");
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "llmfit-desktop"
|
||||
version.workspace = true
|
||||
edition = "2024"
|
||||
authors = ["axjns"]
|
||||
description = "macOS desktop application for llmfit — visual LLM hardware fitting"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/AlexsJones/llmfit"
|
||||
default-run = "llmfit-desktop"
|
||||
|
||||
[[bin]]
|
||||
name = "llmfit-desktop"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
llmfit-core = { path = "../llmfit-core" }
|
||||
tauri = { version = "2", features = [] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2", features = [] }
|
||||
|
||||
[features]
|
||||
default = ["custom-protocol"]
|
||||
custom-protocol = ["tauri/custom-protocol"]
|
||||
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
tauri_build::build()
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"identifier": "default",
|
||||
"description": "Default permissions for llmfit desktop",
|
||||
"windows": ["main"],
|
||||
"permissions": ["core:default"]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,206 @@
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
use llmfit_core::fit::{FitLevel, InferenceRuntime, ModelFit, RunMode};
|
||||
use llmfit_core::hardware::SystemSpecs;
|
||||
use llmfit_core::models::ModelDatabase;
|
||||
use llmfit_core::providers::{ModelProvider, OllamaProvider, PullEvent};
|
||||
use serde::Serialize;
|
||||
use std::sync::Mutex;
|
||||
use tauri::State;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GpuInfoJs {
|
||||
name: String,
|
||||
vram_gb: Option<f64>,
|
||||
backend: String,
|
||||
count: u32,
|
||||
unified_memory: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SystemInfo {
|
||||
total_ram_gb: f64,
|
||||
available_ram_gb: f64,
|
||||
cpu_name: String,
|
||||
cpu_cores: usize,
|
||||
gpus: Vec<GpuInfoJs>,
|
||||
unified_memory: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
struct ModelFitInfo {
|
||||
name: String,
|
||||
params_b: f64,
|
||||
quant: String,
|
||||
fit_level: String,
|
||||
run_mode: String,
|
||||
score: f64,
|
||||
memory_required_gb: f64,
|
||||
memory_available_gb: f64,
|
||||
utilization_pct: f64,
|
||||
estimated_tps: f64,
|
||||
use_case: String,
|
||||
runtime: String,
|
||||
installed: bool,
|
||||
notes: Vec<String>,
|
||||
release_date: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PullStatus {
|
||||
status: String,
|
||||
percent: Option<f64>,
|
||||
done: bool,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
struct AppState {
|
||||
ollama: OllamaProvider,
|
||||
pull_handle: Mutex<Option<llmfit_core::providers::PullHandle>>,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn get_system_specs() -> Result<SystemInfo, String> {
|
||||
let specs = SystemSpecs::detect();
|
||||
let gpus = specs
|
||||
.gpus
|
||||
.iter()
|
||||
.map(|g| GpuInfoJs {
|
||||
name: g.name.clone(),
|
||||
vram_gb: g.vram_gb,
|
||||
backend: format!("{:?}", g.backend),
|
||||
count: g.count,
|
||||
unified_memory: g.unified_memory,
|
||||
})
|
||||
.collect();
|
||||
Ok(SystemInfo {
|
||||
total_ram_gb: specs.total_ram_gb,
|
||||
available_ram_gb: specs.available_ram_gb,
|
||||
cpu_name: specs.cpu_name.clone(),
|
||||
cpu_cores: specs.total_cpu_cores,
|
||||
gpus,
|
||||
unified_memory: specs.unified_memory,
|
||||
})
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn get_model_fits() -> Result<Vec<ModelFitInfo>, String> {
|
||||
let specs = SystemSpecs::detect();
|
||||
let db = ModelDatabase::new();
|
||||
|
||||
let mut fits: Vec<ModelFit> = db
|
||||
.get_all_models()
|
||||
.iter()
|
||||
.map(|m| ModelFit::analyze(m, &specs))
|
||||
.collect();
|
||||
|
||||
fits = llmfit_core::fit::rank_models_by_fit(fits);
|
||||
|
||||
Ok(fits
|
||||
.into_iter()
|
||||
.map(|f| ModelFitInfo {
|
||||
name: f.model.name.clone(),
|
||||
params_b: f.model.parameters_raw.unwrap_or(0) as f64 / 1e9,
|
||||
quant: f.best_quant.clone(),
|
||||
fit_level: match f.fit_level {
|
||||
FitLevel::Perfect => "Perfect".to_string(),
|
||||
FitLevel::Good => "Good".to_string(),
|
||||
FitLevel::Marginal => "Marginal".to_string(),
|
||||
FitLevel::TooTight => "Too Tight".to_string(),
|
||||
},
|
||||
run_mode: match f.run_mode {
|
||||
RunMode::Gpu => "GPU".to_string(),
|
||||
RunMode::CpuOffload => "CPU Offload".to_string(),
|
||||
RunMode::CpuOnly => "CPU Only".to_string(),
|
||||
RunMode::MoeOffload => "MoE Offload".to_string(),
|
||||
RunMode::TensorParallel => "Tensor Parallel".to_string(),
|
||||
},
|
||||
score: f.score,
|
||||
memory_required_gb: f.memory_required_gb,
|
||||
memory_available_gb: f.memory_available_gb,
|
||||
utilization_pct: f.utilization_pct,
|
||||
estimated_tps: f.estimated_tps,
|
||||
use_case: format!("{:?}", f.use_case),
|
||||
runtime: match f.runtime {
|
||||
InferenceRuntime::LlamaCpp => "llama.cpp".to_string(),
|
||||
InferenceRuntime::Mlx => "MLX".to_string(),
|
||||
InferenceRuntime::Vllm => "vLLM".to_string(),
|
||||
InferenceRuntime::Unsupported => "unsupported".to_string(),
|
||||
},
|
||||
installed: f.installed,
|
||||
notes: f.notes.clone(),
|
||||
release_date: f.model.release_date.clone(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn start_pull(model_tag: String, state: State<'_, AppState>) -> Result<String, String> {
|
||||
let handle = state.ollama.start_pull(&model_tag)?;
|
||||
let mut pull = state.pull_handle.lock().map_err(|e| e.to_string())?;
|
||||
*pull = Some(handle);
|
||||
Ok("started".to_string())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn poll_pull(state: State<'_, AppState>) -> Result<PullStatus, String> {
|
||||
let pull = state.pull_handle.lock().map_err(|e| e.to_string())?;
|
||||
if let Some(ref handle) = *pull {
|
||||
match handle.receiver.try_recv() {
|
||||
Ok(PullEvent::Progress { status, percent }) => Ok(PullStatus {
|
||||
status,
|
||||
percent,
|
||||
done: false,
|
||||
error: None,
|
||||
}),
|
||||
Ok(PullEvent::Done) => Ok(PullStatus {
|
||||
status: "Complete".to_string(),
|
||||
percent: Some(100.0),
|
||||
done: true,
|
||||
error: None,
|
||||
}),
|
||||
Ok(PullEvent::Error(e)) => Ok(PullStatus {
|
||||
status: "Error".to_string(),
|
||||
percent: None,
|
||||
done: true,
|
||||
error: Some(e),
|
||||
}),
|
||||
Err(std::sync::mpsc::TryRecvError::Empty) => Ok(PullStatus {
|
||||
status: "Waiting...".to_string(),
|
||||
percent: None,
|
||||
done: false,
|
||||
error: None,
|
||||
}),
|
||||
Err(std::sync::mpsc::TryRecvError::Disconnected) => Ok(PullStatus {
|
||||
status: "Complete".to_string(),
|
||||
percent: Some(100.0),
|
||||
done: true,
|
||||
error: None,
|
||||
}),
|
||||
}
|
||||
} else {
|
||||
Err("No pull in progress".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn is_ollama_available(state: State<'_, AppState>) -> bool {
|
||||
state.ollama.is_available()
|
||||
}
|
||||
|
||||
fn main() {
|
||||
tauri::Builder::default()
|
||||
.manage(AppState {
|
||||
ollama: OllamaProvider::new(),
|
||||
pull_handle: Mutex::new(None),
|
||||
})
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
get_system_specs,
|
||||
get_model_fits,
|
||||
start_pull,
|
||||
poll_pull,
|
||||
is_ollama_available,
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
.expect("error while running tauri application");
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"productName": "llmfit",
|
||||
"version": "0.4.8",
|
||||
"identifier": "com.llmfit.desktop",
|
||||
"build": {
|
||||
"frontendDist": "./ui"
|
||||
},
|
||||
"bundle": {
|
||||
"windows": {
|
||||
"webviewInstallMode": "downloadBootstrapper"
|
||||
}
|
||||
},
|
||||
"app": {
|
||||
"withGlobalTauri": true,
|
||||
"windows": [
|
||||
{
|
||||
"title": "llmfit — LLM Hardware Fitting",
|
||||
"width": 1200,
|
||||
"height": 800,
|
||||
"resizable": true
|
||||
}
|
||||
],
|
||||
"security": {
|
||||
"csp": "default-src 'self'; style-src 'self'; script-src 'self'"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
const invoke = window.__TAURI_INTERNALS__
|
||||
? window.__TAURI_INTERNALS__.invoke
|
||||
: async (cmd) => { console.warn('Tauri not available, cmd:', cmd); return null; };
|
||||
|
||||
const {
|
||||
t,
|
||||
getLocale,
|
||||
setLocale,
|
||||
subscribe,
|
||||
applyStaticTranslations,
|
||||
translateFitLevel,
|
||||
translateRunMode,
|
||||
translateUseCase,
|
||||
} = window.llmfitI18n;
|
||||
|
||||
let allFits = [];
|
||||
let ollamaAvailable = false;
|
||||
let pullInterval = null;
|
||||
let lastSpecs = null;
|
||||
let currentModalFit = null;
|
||||
|
||||
function esc(s) {
|
||||
const d = document.createElement('div');
|
||||
d.textContent = s;
|
||||
return d.innerHTML;
|
||||
}
|
||||
|
||||
function renderSpecs(specs) {
|
||||
if (!specs) return;
|
||||
|
||||
document.getElementById('cpu-name').textContent = specs.cpu_name;
|
||||
document.getElementById('cpu-cores').textContent = t('system.cores', { count: specs.cpu_cores });
|
||||
document.getElementById('ram-total').textContent = specs.total_ram_gb.toFixed(1) + ' GB';
|
||||
document.getElementById('ram-available').textContent = specs.available_ram_gb.toFixed(1) + ' GB';
|
||||
|
||||
const container = document.getElementById('gpus-container');
|
||||
container.innerHTML = '';
|
||||
|
||||
if (specs.gpus.length === 0) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'spec-card';
|
||||
card.innerHTML = '<span class="spec-label">' + esc(t('system.gpu')) + '</span>' +
|
||||
'<span class="spec-value">' + esc(t('system.noGpu')) + '</span>';
|
||||
container.appendChild(card);
|
||||
} else {
|
||||
specs.gpus.forEach((gpu, i) => {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'spec-card';
|
||||
const label = specs.gpus.length > 1 ? t('system.gpuIndexed', { index: i + 1 }) : t('system.gpu');
|
||||
const countStr = gpu.count > 1 ? ' ×' + gpu.count : '';
|
||||
const vramStr = gpu.vram_gb != null ? gpu.vram_gb.toFixed(1) + ' GB VRAM' : t('system.sharedMemory');
|
||||
const backendStr = gpu.backend !== 'None' ? gpu.backend : '';
|
||||
const details = [vramStr, backendStr].filter(Boolean).join(' · ');
|
||||
card.innerHTML = '<span class="spec-label">' + esc(label) + '</span>' +
|
||||
'<span class="spec-value">' + esc(gpu.name + countStr) + '</span>' +
|
||||
'<span class="spec-detail">' + esc(details) + '</span>';
|
||||
container.appendChild(card);
|
||||
});
|
||||
}
|
||||
|
||||
if (specs.unified_memory) {
|
||||
const archCard = document.getElementById('memory-arch-card');
|
||||
archCard.style.display = '';
|
||||
document.getElementById('memory-arch').textContent = t('system.unifiedMemory');
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSpecs() {
|
||||
try {
|
||||
const specs = await invoke('get_system_specs');
|
||||
if (!specs) return;
|
||||
lastSpecs = specs;
|
||||
renderSpecs(specs);
|
||||
} catch (e) {
|
||||
console.error('Failed to load specs:', e);
|
||||
document.getElementById('cpu-name').textContent = t('system.errorLoading');
|
||||
}
|
||||
}
|
||||
|
||||
function fitClass(level) {
|
||||
switch (level) {
|
||||
case 'Perfect': return 'fit-perfect';
|
||||
case 'Good': return 'fit-good';
|
||||
case 'Marginal': return 'fit-marginal';
|
||||
default: return 'fit-tight';
|
||||
}
|
||||
}
|
||||
|
||||
function modeClass(mode) {
|
||||
switch (mode) {
|
||||
case 'GPU': return 'mode-gpu';
|
||||
case 'MoE Offload': return 'mode-moe';
|
||||
case 'CPU Offload': return 'mode-cpuoffload';
|
||||
default: return 'mode-cpuonly';
|
||||
}
|
||||
}
|
||||
|
||||
function showModal(fit) {
|
||||
currentModalFit = fit;
|
||||
const modal = document.getElementById('model-modal');
|
||||
const body = document.getElementById('modal-body');
|
||||
|
||||
const memBar = Math.min(fit.utilization_pct, 100);
|
||||
const memBarClass = fit.utilization_pct > 95 ? 'bar-red' : fit.utilization_pct > 80 ? 'bar-yellow' : 'bar-green';
|
||||
|
||||
let notesHtml = '';
|
||||
if (fit.notes && fit.notes.length > 0) {
|
||||
notesHtml = '<div class="modal-section"><h4>' + esc(t('desktop.notes')) + '</h4><ul>' +
|
||||
fit.notes.map(n => '<li>' + esc(n) + '</li>').join('') +
|
||||
'</ul></div>';
|
||||
}
|
||||
|
||||
const installedBadge = fit.installed
|
||||
? '<span class="badge badge-installed">' + esc(t('desktop.installed')) + '</span>'
|
||||
: '<span class="badge badge-not-installed">' + esc(t('desktop.notInstalled')) + '</span>';
|
||||
|
||||
const downloadBtn = (!fit.installed && ollamaAvailable)
|
||||
? '<button class="btn-download">' + esc(t('desktop.downloadViaOllama')) + '</button>'
|
||||
: '';
|
||||
|
||||
body.innerHTML = `
|
||||
<div class="modal-header-row">
|
||||
<h3>${esc(fit.name)}</h3>
|
||||
${installedBadge}
|
||||
</div>
|
||||
|
||||
<div class="modal-grid">
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.parameters'))}</span>
|
||||
<span class="stat-value">${esc(fit.params_b.toFixed(1))}B</span>
|
||||
</div>
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.quantization'))}</span>
|
||||
<span class="stat-value">${esc(fit.quant)}</span>
|
||||
</div>
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.runtime'))}</span>
|
||||
<span class="stat-value">${esc(fit.runtime)}</span>
|
||||
</div>
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.score'))}</span>
|
||||
<span class="stat-value">${esc(fit.score.toFixed(0))}/100</span>
|
||||
</div>
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.estSpeed'))}</span>
|
||||
<span class="stat-value">${esc(fit.estimated_tps.toFixed(1))} tok/s</span>
|
||||
</div>
|
||||
<div class="modal-stat">
|
||||
<span class="stat-label">${esc(t('desktop.useCase'))}</span>
|
||||
<span class="stat-value">${esc(translateUseCase(fit.use_case))}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal-section">
|
||||
<h4>${esc(t('desktop.fitAnalysis'))}</h4>
|
||||
<div class="fit-row">
|
||||
<span class="${fitClass(fit.fit_level)}">${esc(translateFitLevel(fit.fit_level))}</span>
|
||||
<span class="fit-detail">${esc(translateRunMode(fit.run_mode))}</span>
|
||||
</div>
|
||||
<div class="mem-bar-container">
|
||||
<div class="mem-bar-label">
|
||||
<span>${esc(t('desktop.memorySummary', { required: fit.memory_required_gb.toFixed(1), available: fit.memory_available_gb.toFixed(1) }))}</span>
|
||||
<span>${esc(fit.utilization_pct.toFixed(0))}%</span>
|
||||
</div>
|
||||
<div class="mem-bar-track">
|
||||
<div class="mem-bar-fill ${memBarClass}" style="width: ${memBar}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
${notesHtml}
|
||||
|
||||
<div id="pull-status" class="pull-status" style="display:none">
|
||||
<div class="pull-status-text"></div>
|
||||
<div class="mem-bar-track">
|
||||
<div class="pull-bar-fill" style="width: 0%"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal-actions">
|
||||
${downloadBtn}
|
||||
<button class="btn-close" onclick="closeModal()">${esc(t('desktop.close'))}</button>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const dlBtn = body.querySelector('.btn-download');
|
||||
if (dlBtn) dlBtn.addEventListener('click', () => pullModel(fit.name));
|
||||
|
||||
modal.classList.add('visible');
|
||||
}
|
||||
|
||||
function closeModal() {
|
||||
currentModalFit = null;
|
||||
document.getElementById('model-modal').classList.remove('visible');
|
||||
if (pullInterval) {
|
||||
clearInterval(pullInterval);
|
||||
pullInterval = null;
|
||||
}
|
||||
}
|
||||
window.closeModal = closeModal;
|
||||
|
||||
async function pullModel(name) {
|
||||
const statusEl = document.getElementById('pull-status');
|
||||
const textEl = statusEl.querySelector('.pull-status-text');
|
||||
const barEl = statusEl.querySelector('.pull-bar-fill');
|
||||
const btn = document.querySelector('.btn-download');
|
||||
|
||||
statusEl.style.display = '';
|
||||
if (btn) btn.disabled = true;
|
||||
textEl.textContent = t('desktop.startingDownload');
|
||||
|
||||
try {
|
||||
await invoke('start_pull', { modelTag: name });
|
||||
|
||||
pullInterval = setInterval(async () => {
|
||||
try {
|
||||
const s = await invoke('poll_pull');
|
||||
if (!s) return;
|
||||
textEl.textContent = s.status;
|
||||
if (s.percent != null) barEl.style.width = s.percent + '%';
|
||||
if (s.done) {
|
||||
clearInterval(pullInterval);
|
||||
pullInterval = null;
|
||||
if (s.error) {
|
||||
textEl.textContent = t('desktop.errorPrefix') + s.error;
|
||||
if (btn) btn.disabled = false;
|
||||
} else {
|
||||
textEl.textContent = t('desktop.downloadComplete');
|
||||
barEl.style.width = '100%';
|
||||
await loadModels();
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Poll error:', e);
|
||||
}
|
||||
}, 500);
|
||||
} catch (e) {
|
||||
textEl.textContent = t('desktop.errorPrefix') + e;
|
||||
if (btn) btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
function renderModels(fits) {
|
||||
const tbody = document.getElementById('models-body');
|
||||
if (!fits || fits.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="9" class="loading">' + esc(t('desktop.noModels')) + '</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = fits.map((f, i) => `
|
||||
<tr class="model-row" data-index="${i}">
|
||||
<td><strong>${esc(f.name)}</strong>${f.installed ? ' <span class="installed-dot" title="' + esc(t('desktop.installed')) + '">●</span>' : ''}</td>
|
||||
<td>${esc(f.params_b.toFixed(1))}B</td>
|
||||
<td>${esc(f.quant)}</td>
|
||||
<td class="${fitClass(f.fit_level)}">${esc(translateFitLevel(f.fit_level))}</td>
|
||||
<td class="${modeClass(f.run_mode)}">${esc(translateRunMode(f.run_mode))}</td>
|
||||
<td>${esc(f.score.toFixed(0))}</td>
|
||||
<td>${esc(f.memory_required_gb.toFixed(1))} GB</td>
|
||||
<td>${esc(f.estimated_tps.toFixed(1))}</td>
|
||||
<td>${esc(translateUseCase(f.use_case))}</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
|
||||
const currentFits = fits;
|
||||
tbody.querySelectorAll('.model-row').forEach(row => {
|
||||
row.addEventListener('click', () => {
|
||||
const idx = parseInt(row.dataset.index, 10);
|
||||
showModal(currentFits[idx]);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function applyFilters() {
|
||||
const search = document.getElementById('search').value.toLowerCase();
|
||||
const fitFilter = document.getElementById('fit-filter').value;
|
||||
|
||||
let filtered = allFits;
|
||||
if (search) {
|
||||
filtered = filtered.filter(f => f.name.toLowerCase().includes(search));
|
||||
}
|
||||
if (fitFilter !== 'all') {
|
||||
filtered = filtered.filter(f => f.fit_level === fitFilter);
|
||||
}
|
||||
renderModels(filtered);
|
||||
}
|
||||
|
||||
async function loadModels() {
|
||||
try {
|
||||
allFits = await invoke('get_model_fits') || [];
|
||||
applyFilters();
|
||||
} catch (e) {
|
||||
console.error('Failed to load models:', e);
|
||||
document.getElementById('models-body').innerHTML =
|
||||
'<tr><td colspan="9" class="loading">' + esc(t('desktop.errorLoadingModels')) + '</td></tr>';
|
||||
}
|
||||
}
|
||||
|
||||
function rerenderForLocale() {
|
||||
applyStaticTranslations();
|
||||
document.getElementById('locale-select').value = getLocale();
|
||||
if (lastSpecs) {
|
||||
renderSpecs(lastSpecs);
|
||||
}
|
||||
applyFilters();
|
||||
if (currentModalFit) {
|
||||
showModal(currentModalFit);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('model-modal').addEventListener('click', (e) => {
|
||||
if (e.target === e.currentTarget) closeModal();
|
||||
});
|
||||
|
||||
document.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape') closeModal();
|
||||
});
|
||||
|
||||
document.getElementById('search').addEventListener('input', applyFilters);
|
||||
document.getElementById('fit-filter').addEventListener('change', applyFilters);
|
||||
document.getElementById('locale-select').addEventListener('change', (e) => {
|
||||
setLocale(e.target.value);
|
||||
});
|
||||
|
||||
subscribe(rerenderForLocale);
|
||||
|
||||
async function init() {
|
||||
applyStaticTranslations();
|
||||
document.getElementById('locale-select').value = getLocale();
|
||||
ollamaAvailable = await invoke('is_ollama_available') || false;
|
||||
loadSpecs();
|
||||
loadModels();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -0,0 +1,322 @@
|
||||
(function initI18n(global) {
|
||||
const LOCALE_KEY = 'llmfit.locale';
|
||||
const FALLBACK_LOCALE = 'en';
|
||||
|
||||
const MESSAGES = {
|
||||
en: {
|
||||
language: {
|
||||
label: 'Language',
|
||||
english: 'English',
|
||||
chinese: '中文'
|
||||
},
|
||||
system: {
|
||||
title: 'System',
|
||||
cpu: 'CPU',
|
||||
totalRam: 'Total RAM',
|
||||
availableRam: 'Available RAM',
|
||||
memory: 'Memory',
|
||||
gpu: 'GPU',
|
||||
detecting: 'Detecting…',
|
||||
noGpu: 'No GPU detected',
|
||||
sharedMemory: 'Shared memory',
|
||||
unifiedMemory: 'Unified (CPU + GPU shared)',
|
||||
errorLoading: 'Error loading specs',
|
||||
cores: ({ count }) => `${count} cores`,
|
||||
gpuIndexed: ({ index }) => `GPU ${index}`
|
||||
},
|
||||
desktop: {
|
||||
pageTitle: 'llmfit',
|
||||
modelsTitle: 'Model Compatibility',
|
||||
searchPlaceholder: 'Filter models...',
|
||||
allFitLevels: 'All Fit Levels',
|
||||
loadingModels: 'Loading models...',
|
||||
noModels: 'No models found',
|
||||
errorLoadingModels: 'Error loading models',
|
||||
notes: 'Notes',
|
||||
fitAnalysis: 'Fit Analysis',
|
||||
installed: 'Installed',
|
||||
notInstalled: 'Not Installed',
|
||||
downloadViaOllama: '⬇ Download via Ollama',
|
||||
close: 'Close',
|
||||
parameters: 'Parameters',
|
||||
quantization: 'Quantization',
|
||||
runtime: 'Runtime',
|
||||
score: 'Score',
|
||||
estSpeed: 'Est. Speed',
|
||||
useCase: 'Use Case',
|
||||
memorySummary: ({ required, available }) => `Memory: ${required} / ${available} GB`,
|
||||
startingDownload: 'Starting download...',
|
||||
downloadComplete: 'Download complete!',
|
||||
errorPrefix: 'Error: '
|
||||
},
|
||||
table: {
|
||||
model: 'Model',
|
||||
params: 'Params',
|
||||
quant: 'Quant',
|
||||
fit: 'Fit',
|
||||
mode: 'Mode',
|
||||
score: 'Score',
|
||||
ramReq: 'RAM Req',
|
||||
estTps: 'Est. TPS',
|
||||
useCase: 'Use Case'
|
||||
},
|
||||
labels: {
|
||||
fit: {
|
||||
perfect: 'Perfect',
|
||||
good: 'Good',
|
||||
marginal: 'Marginal',
|
||||
too_tight: 'Too Tight'
|
||||
},
|
||||
runMode: {
|
||||
gpu: 'GPU',
|
||||
moe_offload: 'MoE Offload',
|
||||
cpu_offload: 'CPU Offload',
|
||||
cpu_only: 'CPU Only'
|
||||
},
|
||||
useCase: {
|
||||
general: 'General',
|
||||
coding: 'Coding',
|
||||
reasoning: 'Reasoning',
|
||||
chat: 'Chat',
|
||||
multimodal: 'Multimodal',
|
||||
embedding: 'Embedding'
|
||||
}
|
||||
}
|
||||
},
|
||||
'zh-CN': {
|
||||
language: {
|
||||
label: '语言',
|
||||
english: 'English',
|
||||
chinese: '中文'
|
||||
},
|
||||
system: {
|
||||
title: '系统信息',
|
||||
cpu: 'CPU',
|
||||
totalRam: '总内存',
|
||||
availableRam: '可用内存',
|
||||
memory: '内存',
|
||||
gpu: 'GPU',
|
||||
detecting: '检测中…',
|
||||
noGpu: '未检测到 GPU',
|
||||
sharedMemory: '共享内存',
|
||||
unifiedMemory: '统一内存（CPU 与 GPU 共享）',
|
||||
errorLoading: '加载硬件信息失败',
|
||||
cores: ({ count }) => `${count} 核`,
|
||||
gpuIndexed: ({ index }) => `GPU ${index}`
|
||||
},
|
||||
desktop: {
|
||||
pageTitle: 'llmfit',
|
||||
modelsTitle: '模型适配分析',
|
||||
searchPlaceholder: '筛选模型...',
|
||||
allFitLevels: '全部适配等级',
|
||||
loadingModels: '正在加载模型...',
|
||||
noModels: '未找到匹配模型',
|
||||
errorLoadingModels: '加载模型失败',
|
||||
notes: '说明',
|
||||
fitAnalysis: '适配分析',
|
||||
installed: '已安装',
|
||||
notInstalled: '未安装',
|
||||
downloadViaOllama: '⬇ 通过 Ollama 下载',
|
||||
close: '关闭',
|
||||
parameters: '参数量',
|
||||
quantization: '量化',
|
||||
runtime: '运行时',
|
||||
score: '得分',
|
||||
estSpeed: '预估速度',
|
||||
useCase: '用途',
|
||||
memorySummary: ({ required, available }) => `内存：${required} / ${available} GB`,
|
||||
startingDownload: '开始下载...',
|
||||
downloadComplete: '下载完成！',
|
||||
errorPrefix: '错误：'
|
||||
},
|
||||
table: {
|
||||
model: '模型',
|
||||
params: '参数量',
|
||||
quant: '量化',
|
||||
fit: '适配度',
|
||||
mode: '模式',
|
||||
score: '得分',
|
||||
ramReq: '内存需求',
|
||||
estTps: '预估 TPS',
|
||||
useCase: '用途'
|
||||
},
|
||||
labels: {
|
||||
fit: {
|
||||
perfect: '完美适配',
|
||||
good: '良好适配',
|
||||
marginal: '勉强可用',
|
||||
too_tight: '过紧无法稳定运行'
|
||||
},
|
||||
runMode: {
|
||||
gpu: 'GPU',
|
||||
moe_offload: 'MoE 卸载',
|
||||
cpu_offload: 'CPU 卸载',
|
||||
cpu_only: '仅 CPU'
|
||||
},
|
||||
useCase: {
|
||||
general: '通用',
|
||||
coding: '编程',
|
||||
reasoning: '推理',
|
||||
chat: '对话',
|
||||
multimodal: '多模态',
|
||||
embedding: '向量嵌入'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function getNestedValue(obj, key) {
|
||||
return key.split('.').reduce((acc, part) => (acc ? acc[part] : undefined), obj);
|
||||
}
|
||||
|
||||
function formatMessage(message, params) {
|
||||
if (typeof message === 'function') {
|
||||
return message(params || {});
|
||||
}
|
||||
if (typeof message !== 'string') {
|
||||
return message;
|
||||
}
|
||||
return message.replace(/\{(\w+)\}/g, function replaceToken(_, token) {
|
||||
return params && params[token] != null ? String(params[token]) : `{${token}}`;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeLocale(locale) {
|
||||
if (!locale || typeof locale !== 'string') {
|
||||
return FALLBACK_LOCALE;
|
||||
}
|
||||
return locale.toLowerCase().startsWith('zh') ? 'zh-CN' : FALLBACK_LOCALE;
|
||||
}
|
||||
|
||||
function getStoredLocale() {
|
||||
try {
|
||||
const stored = global.localStorage.getItem(LOCALE_KEY);
|
||||
return stored ? normalizeLocale(stored) : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function detectLocale() {
|
||||
return getStoredLocale() || normalizeLocale(global.navigator && global.navigator.language);
|
||||
}
|
||||
|
||||
let currentLocale = detectLocale();
|
||||
const listeners = new Set();
|
||||
|
||||
function t(key, params) {
|
||||
const value =
|
||||
getNestedValue(MESSAGES[currentLocale], key) ??
|
||||
getNestedValue(MESSAGES[FALLBACK_LOCALE], key) ??
|
||||
key;
|
||||
return formatMessage(value, params);
|
||||
}
|
||||
|
||||
function setLocale(locale) {
|
||||
const nextLocale = normalizeLocale(locale);
|
||||
if (nextLocale === currentLocale) {
|
||||
return;
|
||||
}
|
||||
currentLocale = nextLocale;
|
||||
try {
|
||||
global.localStorage.setItem(LOCALE_KEY, currentLocale);
|
||||
} catch (_) {
|
||||
// ignore storage failures
|
||||
}
|
||||
document.documentElement.lang = currentLocale;
|
||||
listeners.forEach(function notify(listener) {
|
||||
listener(currentLocale);
|
||||
});
|
||||
}
|
||||
|
||||
function getLocale() {
|
||||
return currentLocale;
|
||||
}
|
||||
|
||||
function subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return function unsubscribe() {
|
||||
listeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
function applyStaticTranslations() {
|
||||
document.title = t('desktop.pageTitle');
|
||||
document.documentElement.lang = currentLocale;
|
||||
|
||||
document.querySelectorAll('[data-i18n]').forEach(function updateText(node) {
|
||||
node.textContent = t(node.getAttribute('data-i18n'));
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-placeholder]').forEach(function updatePlaceholder(node) {
|
||||
node.setAttribute('placeholder', t(node.getAttribute('data-i18n-placeholder')));
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-aria-label]').forEach(function updateAria(node) {
|
||||
node.setAttribute('aria-label', t(node.getAttribute('data-i18n-aria-label')));
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeFitCode(value) {
|
||||
if (!value) return null;
|
||||
const normalized = String(value).trim().toLowerCase().replace(/[\s-]+/g, '_');
|
||||
const aliases = {
|
||||
perfect: 'perfect',
|
||||
good: 'good',
|
||||
marginal: 'marginal',
|
||||
too_tight: 'too_tight',
|
||||
tootight: 'too_tight'
|
||||
};
|
||||
return aliases[normalized] || null;
|
||||
}
|
||||
|
||||
function normalizeRunModeCode(value) {
|
||||
if (!value) return null;
|
||||
const normalized = String(value).trim().toLowerCase().replace(/[\s-]+/g, '_');
|
||||
const aliases = {
|
||||
gpu: 'gpu',
|
||||
moe_offload: 'moe_offload',
|
||||
cpu_offload: 'cpu_offload',
|
||||
cpu_only: 'cpu_only'
|
||||
};
|
||||
return aliases[normalized] || null;
|
||||
}
|
||||
|
||||
function normalizeUseCaseCode(value) {
|
||||
if (!value) return null;
|
||||
const normalized = String(value).trim().toLowerCase();
|
||||
return ['general', 'coding', 'reasoning', 'chat', 'multimodal', 'embedding'].includes(normalized)
|
||||
? normalized
|
||||
: null;
|
||||
}
|
||||
|
||||
function translateFitLevel(value) {
|
||||
const code = normalizeFitCode(value);
|
||||
return code ? t(`labels.fit.${code}`) : (value || '—');
|
||||
}
|
||||
|
||||
function translateRunMode(value) {
|
||||
const code = normalizeRunModeCode(value);
|
||||
return code ? t(`labels.runMode.${code}`) : (value || '—');
|
||||
}
|
||||
|
||||
function translateUseCase(value) {
|
||||
const code = normalizeUseCaseCode(value);
|
||||
return code ? t(`labels.useCase.${code}`) : (value || '—');
|
||||
}
|
||||
|
||||
global.llmfitI18n = {
|
||||
LOCALE_KEY,
|
||||
getLocale,
|
||||
setLocale,
|
||||
subscribe,
|
||||
t,
|
||||
applyStaticTranslations,
|
||||
normalizeFitCode,
|
||||
normalizeRunModeCode,
|
||||
normalizeUseCaseCode,
|
||||
translateFitLevel,
|
||||
translateRunMode,
|
||||
translateUseCase
|
||||
};
|
||||
})(window);
|
||||
@@ -0,0 +1,81 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>llmfit</title>
|
||||
<link rel="stylesheet" href="styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<section id="system-panel">
|
||||
<h2 data-i18n="system.title">System</h2>
|
||||
<div id="specs-grid" class="specs-grid">
|
||||
<div class="spec-card">
|
||||
<span class="spec-label" data-i18n="system.cpu">CPU</span>
|
||||
<span id="cpu-name" class="spec-value" data-i18n="system.detecting">Detecting...</span>
|
||||
<span id="cpu-cores" class="spec-detail"></span>
|
||||
</div>
|
||||
<div class="spec-card">
|
||||
<span class="spec-label" data-i18n="system.totalRam">Total RAM</span>
|
||||
<span id="ram-total" class="spec-value">—</span>
|
||||
</div>
|
||||
<div class="spec-card">
|
||||
<span class="spec-label" data-i18n="system.availableRam">Available RAM</span>
|
||||
<span id="ram-available" class="spec-value">—</span>
|
||||
</div>
|
||||
<div id="gpus-container">
|
||||
<!-- GPU cards injected by JS -->
|
||||
</div>
|
||||
<div class="spec-card" id="memory-arch-card" style="display:none">
|
||||
<span class="spec-label" data-i18n="system.memory">Memory</span>
|
||||
<span id="memory-arch" class="spec-value">—</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="models-panel">
|
||||
<h2 data-i18n="desktop.modelsTitle">Model Compatibility</h2>
|
||||
<div class="controls">
|
||||
<input type="text" id="search" data-i18n-placeholder="desktop.searchPlaceholder" placeholder="Filter models..." />
|
||||
<select id="fit-filter">
|
||||
<option value="all" data-i18n="desktop.allFitLevels">All Fit Levels</option>
|
||||
<option value="Perfect" data-i18n="labels.fit.perfect">Perfect</option>
|
||||
<option value="Good" data-i18n="labels.fit.good">Good</option>
|
||||
<option value="Marginal" data-i18n="labels.fit.marginal">Marginal</option>
|
||||
<option value="Too Tight" data-i18n="labels.fit.too_tight">Too Tight</option>
|
||||
</select>
|
||||
<select id="locale-select" data-i18n-aria-label="language.label" aria-label="Language">
|
||||
<option value="en" data-i18n="language.english">English</option>
|
||||
<option value="zh-CN" data-i18n="language.chinese">中文</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="models-table-container">
|
||||
<table id="models-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th data-i18n="table.model">Model</th>
|
||||
<th data-i18n="table.params">Params</th>
|
||||
<th data-i18n="table.quant">Quant</th>
|
||||
<th data-i18n="table.fit">Fit</th>
|
||||
<th data-i18n="table.mode">Mode</th>
|
||||
<th data-i18n="table.score">Score</th>
|
||||
<th data-i18n="table.ramReq">RAM Req</th>
|
||||
<th data-i18n="table.estTps">Est. TPS</th>
|
||||
<th data-i18n="table.useCase">Use Case</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="models-body">
|
||||
<tr><td colspan="9" class="loading" data-i18n="desktop.loadingModels">Loading models...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div id="model-modal" class="modal-overlay">
|
||||
<div class="modal-content" id="modal-body"></div>
|
||||
</div>
|
||||
|
||||
<script src="i18n.js"></script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,276 @@
|
||||
:root {
|
||||
--bg: #0d1117;
|
||||
--surface: #161b22;
|
||||
--border: #30363d;
|
||||
--text: #e6edf3;
|
||||
--text-dim: #8b949e;
|
||||
--accent: #58a6ff;
|
||||
--green: #3fb950;
|
||||
--yellow: #d29922;
|
||||
--red: #f85149;
|
||||
--cyan: #39c5cf;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
padding: 24px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
h2 { font-size: 18px; font-weight: 600; margin-bottom: 12px; color: var(--text-dim); }
|
||||
|
||||
.specs-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
||||
gap: 12px;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
#gpus-container {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.spec-card {
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.spec-label { font-size: 12px; text-transform: uppercase; color: var(--text-dim); margin-bottom: 4px; }
|
||||
.spec-value { font-size: 16px; font-weight: 600; }
|
||||
.spec-detail { font-size: 13px; color: var(--text-dim); margin-top: 2px; }
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
margin-bottom: 12px;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
#search {
|
||||
flex: 1;
|
||||
padding: 8px 12px;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
color: var(--text);
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
#fit-filter,
|
||||
#locale-select {
|
||||
padding: 8px 12px;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
color: var(--text);
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
#models-table-container {
|
||||
overflow-x: auto;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
th {
|
||||
text-align: left;
|
||||
padding: 10px 12px;
|
||||
background: var(--surface);
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-weight: 600;
|
||||
color: var(--text-dim);
|
||||
position: sticky;
|
||||
top: 0;
|
||||
}
|
||||
|
||||
td {
|
||||
padding: 8px 12px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
tr:hover { background: var(--surface); }
|
||||
|
||||
.loading { text-align: center; color: var(--text-dim); padding: 32px; }
|
||||
|
||||
.fit-perfect { color: var(--green); font-weight: 600; }
|
||||
.fit-good { color: var(--accent); font-weight: 600; }
|
||||
.fit-marginal { color: var(--yellow); }
|
||||
.fit-tight { color: var(--red); }
|
||||
|
||||
.mode-gpu { color: var(--green); }
|
||||
.mode-moe { color: var(--cyan); }
|
||||
.mode-cpuoffload { color: var(--yellow); }
|
||||
.mode-cpuonly { color: var(--text-dim); }
|
||||
|
||||
/* Clickable rows */
|
||||
.model-row { cursor: pointer; }
|
||||
.model-row:hover { background: var(--surface) !important; }
|
||||
|
||||
.installed-dot { color: var(--green); font-size: 10px; }
|
||||
|
||||
/* Modal */
|
||||
.modal-overlay {
|
||||
display: none;
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
backdrop-filter: blur(4px);
|
||||
z-index: 100;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.modal-overlay.visible { display: flex; }
|
||||
|
||||
.modal-content {
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 12px;
|
||||
padding: 24px;
|
||||
max-width: 560px;
|
||||
width: 90%;
|
||||
max-height: 80vh;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.modal-header-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.modal-header-row h3 { font-size: 20px; font-weight: 700; }
|
||||
|
||||
.badge {
|
||||
font-size: 11px;
|
||||
padding: 2px 8px;
|
||||
border-radius: 12px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.badge-installed { background: rgba(63, 185, 80, 0.15); color: var(--green); }
|
||||
.badge-not-installed { background: rgba(139, 148, 158, 0.15); color: var(--text-dim); }
|
||||
|
||||
.modal-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
gap: 12px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.modal-stat {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.stat-label { font-size: 11px; text-transform: uppercase; color: var(--text-dim); margin-bottom: 2px; }
|
||||
.stat-value { font-size: 15px; font-weight: 600; }
|
||||
|
||||
.modal-section { margin-bottom: 16px; }
|
||||
.modal-section h4 { font-size: 13px; color: var(--text-dim); text-transform: uppercase; margin-bottom: 8px; }
|
||||
.modal-section ul { padding-left: 18px; font-size: 13px; color: var(--text-dim); }
|
||||
.modal-section li { margin-bottom: 4px; }
|
||||
|
||||
.fit-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 8px;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
.fit-detail { color: var(--text-dim); font-size: 13px; }
|
||||
|
||||
.mem-bar-container { margin-top: 4px; }
|
||||
|
||||
.mem-bar-label {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
font-size: 12px;
|
||||
color: var(--text-dim);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.mem-bar-track {
|
||||
height: 8px;
|
||||
background: var(--border);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.mem-bar-fill {
|
||||
height: 100%;
|
||||
border-radius: 4px;
|
||||
transition: width 0.3s;
|
||||
}
|
||||
|
||||
.bar-green { background: var(--green); }
|
||||
.bar-yellow { background: var(--yellow); }
|
||||
.bar-red { background: var(--red); }
|
||||
|
||||
.pull-status {
|
||||
margin: 16px 0;
|
||||
}
|
||||
|
||||
.pull-status-text {
|
||||
font-size: 13px;
|
||||
color: var(--text-dim);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.pull-bar-fill {
|
||||
height: 100%;
|
||||
border-radius: 4px;
|
||||
background: var(--accent);
|
||||
transition: width 0.3s;
|
||||
}
|
||||
|
||||
.modal-actions {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.btn-download {
|
||||
padding: 8px 16px;
|
||||
background: var(--accent);
|
||||
color: var(--bg);
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.btn-download:hover { opacity: 0.9; }
|
||||
.btn-download:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
|
||||
.btn-close {
|
||||
padding: 8px 16px;
|
||||
background: transparent;
|
||||
color: var(--text-dim);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.btn-close:hover { color: var(--text); border-color: var(--text-dim); }
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../LICENSE
|
||||
@@ -0,0 +1,39 @@
|
||||
# Makefile for llmfit-python
|
||||
# Convenience commands for formatting, linting, type-checking, and testing
|
||||
|
||||
.PHONY: help fmt fmt-check lint test typecheck check
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "llmfit-python development targets:"
|
||||
@echo ""
|
||||
@echo " make fmt - Format Python code with ruff"
|
||||
@echo " make fmt-check - Check Python formatting with ruff (no changes)"
|
||||
@echo " make lint - Lint Python code with ruff"
|
||||
@echo " make test - Run Python tests with pytest"
|
||||
@echo " make typecheck - Type-check Python code with ty"
|
||||
@echo " make check - Run all Python quality checks"
|
||||
@echo ""
|
||||
|
||||
# Format Python code
|
||||
fmt:
|
||||
uv run ruff format .
|
||||
|
||||
# Check Python formatting without making changes
|
||||
fmt-check:
|
||||
uv run ruff format --check .
|
||||
|
||||
# Lint Python code
|
||||
lint:
|
||||
uv run ruff check .
|
||||
|
||||
# Run Python tests
|
||||
test:
|
||||
uv run pytest -vv
|
||||
|
||||
# Type-check Python code
|
||||
typecheck:
|
||||
uv run ty check .
|
||||
|
||||
# Run all Python quality checks
|
||||
check: fmt-check lint typecheck test
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user