wehub-resource-sync
1382 lines
44 KiB
Python
1382 lines
44 KiB
Python
"""Tests for CLI helper logic."""
|
|
|
|
import httpx
|
|
import pytest
|
|
from typer import Exit
|
|
|
|
import whichllm.cli as cli_mod
|
|
import whichllm.__main__ as main_mod
|
|
from whichllm.cli import (
|
|
_apply_memory_budgets,
|
|
_apply_gpu_overrides,
|
|
_auto_min_params_for_profile,
|
|
_extract_id_size_b,
|
|
_fill_missing_published_at,
|
|
_format_fetch_error,
|
|
_generate_chat_script,
|
|
_include_vision_candidates,
|
|
_merge_model_eval_benchmarks,
|
|
_parse_memory_amount,
|
|
_pick_gguf_variant,
|
|
_resolve_ranked_gguf_for_run,
|
|
_resolve_evidence_mode,
|
|
_resolve_fit_filter,
|
|
_resolve_speed_filter,
|
|
_search_model,
|
|
_validate_evidence,
|
|
_validate_gpu_flags,
|
|
app,
|
|
)
|
|
from whichllm.utils import _current_version
|
|
from whichllm.engine.types import CompatibilityResult
|
|
from whichllm.hardware.types import GPUInfo, HardwareInfo
|
|
from whichllm.models.types import GGUFVariant, ModelInfo
|
|
from typer.testing import CliRunner
|
|
|
|
|
|
def _hw_with_gpu(vram_gb: int) -> HardwareInfo:
|
|
return HardwareInfo(
|
|
gpus=[
|
|
GPUInfo(
|
|
name="GPU",
|
|
vendor="nvidia",
|
|
vram_bytes=vram_gb * 1024**3,
|
|
memory_bandwidth_gbps=1.0,
|
|
)
|
|
],
|
|
cpu_name="CPU",
|
|
cpu_cores=1,
|
|
ram_bytes=16 * 1024**3,
|
|
disk_free_bytes=100 * 1024**3,
|
|
os="linux",
|
|
)
|
|
|
|
|
|
def test_auto_min_params_general_by_vram():
|
|
# Updated thresholds: tiny GPUs (4-8GB) get a lower floor so they can
|
|
# surface full-GPU 3-4B models instead of being forced into 7B+
|
|
# partial-offload-only candidates.
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(4), "general") == 2.0
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(6), "general") == 3.0
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(8), "general") == 5.0
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(12), "general") == 8.0
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(24), "general") == 10.0
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(32), "general") == 12.0
|
|
|
|
|
|
def test_auto_min_params_non_general_disabled():
|
|
assert _auto_min_params_for_profile(_hw_with_gpu(24), "coding") is None
|
|
|
|
|
|
def test_auto_min_params_uses_usable_vram_budget():
|
|
hw = _hw_with_gpu(20)
|
|
hw.gpus[0].usable_vram_bytes = int(19.0 * 1024**3)
|
|
|
|
assert _auto_min_params_for_profile(hw, "general") == 8.0
|
|
|
|
|
|
def test_auto_min_params_uses_ram_budget_for_shared_memory_gpu():
|
|
hw = HardwareInfo(
|
|
gpus=[
|
|
GPUInfo(
|
|
name="Apple M2",
|
|
vendor="apple",
|
|
vram_bytes=16 * 1024**3,
|
|
usable_vram_bytes=15 * 1024**3,
|
|
shared_memory=True,
|
|
)
|
|
],
|
|
ram_bytes=16 * 1024**3,
|
|
ram_budget_bytes=4 * 1024**3,
|
|
)
|
|
|
|
assert _auto_min_params_for_profile(hw, "general") == 2.0
|
|
|
|
|
|
def test_apply_gpu_overrides_accepts_multiple_simulated_gpus():
|
|
hw = HardwareInfo(gpus=[], ram_bytes=64 * 1024**3, os="linux")
|
|
|
|
_apply_gpu_overrides(hw, cpu_only=False, gpu=["2x RTX 4090"], vram=None)
|
|
|
|
assert len(hw.gpus) == 2
|
|
assert all(gpu.vendor == "nvidia" for gpu in hw.gpus)
|
|
assert all(gpu.vram_bytes == 24 * 1024**3 for gpu in hw.gpus)
|
|
|
|
|
|
def test_validate_gpu_flags_allows_detected_vram_override():
|
|
_validate_gpu_flags(cpu_only=False, gpu=None, vram=8.0, bandwidth=None)
|
|
|
|
|
|
def test_validate_gpu_flags_rejects_non_positive_overrides():
|
|
with pytest.raises(Exit):
|
|
_validate_gpu_flags(cpu_only=False, gpu=None, vram=0, bandwidth=None)
|
|
with pytest.raises(Exit):
|
|
_validate_gpu_flags(cpu_only=False, gpu=None, vram=None, bandwidth=-1)
|
|
|
|
|
|
def test_validate_gpu_flags_rejects_gpu_index_with_simulated_gpu():
|
|
with pytest.raises(Exit):
|
|
_validate_gpu_flags(
|
|
cpu_only=False,
|
|
gpu=["RTX 4090"],
|
|
vram=24.0,
|
|
bandwidth=None,
|
|
gpu_index=0,
|
|
)
|
|
|
|
|
|
def test_apply_gpu_overrides_updates_detected_shared_memory_gpu():
|
|
hw = HardwareInfo(
|
|
gpus=[
|
|
GPUInfo(
|
|
name="Intel UHD Graphics",
|
|
vendor="intel",
|
|
vram_bytes=0,
|
|
shared_memory=True,
|
|
memory_bandwidth_gbps=None,
|
|
)
|
|
],
|
|
cpu_name="CPU",
|
|
cpu_cores=8,
|
|
ram_bytes=32 * 1024**3,
|
|
disk_free_bytes=100 * 1024**3,
|
|
os="linux",
|
|
)
|
|
|
|
_apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=6.0, bandwidth=88.5)
|
|
|
|
assert hw.gpus[0].vram_bytes == 6 * 1024**3
|
|
assert hw.gpus[0].usable_vram_bytes is None
|
|
assert hw.gpus[0].memory_bandwidth_gbps == 88.5
|
|
assert hw.gpus[0].shared_memory is True
|
|
assert hw.gpus[0].vram_overridden is True
|
|
|
|
|
|
def test_apply_gpu_overrides_updates_selected_detected_gpu_only():
|
|
hw = HardwareInfo(
|
|
gpus=[
|
|
GPUInfo(
|
|
name="NVIDIA RTX 4060",
|
|
vendor="nvidia",
|
|
vram_bytes=8 * 1024**3,
|
|
memory_bandwidth_gbps=272.0,
|
|
),
|
|
GPUInfo(
|
|
name="Intel UHD Graphics",
|
|
vendor="intel",
|
|
vram_bytes=0,
|
|
shared_memory=True,
|
|
),
|
|
],
|
|
cpu_name="CPU",
|
|
cpu_cores=8,
|
|
ram_bytes=32 * 1024**3,
|
|
disk_free_bytes=100 * 1024**3,
|
|
os="linux",
|
|
)
|
|
|
|
_apply_gpu_overrides(
|
|
hw, cpu_only=False, gpu=None, vram=4.0, bandwidth=60.0, gpu_index=1
|
|
)
|
|
|
|
assert hw.gpus[0].vram_bytes == 8 * 1024**3
|
|
assert hw.gpus[0].memory_bandwidth_gbps == 272.0
|
|
assert hw.gpus[1].vram_bytes == 4 * 1024**3
|
|
assert hw.gpus[1].memory_bandwidth_gbps == 60.0
|
|
assert hw.gpus[1].vram_overridden is True
|
|
|
|
|
|
def test_apply_gpu_overrides_requires_gpu_index_for_multiple_detected_gpus():
|
|
hw = HardwareInfo(
|
|
gpus=[
|
|
GPUInfo(name="GPU 0", vendor="nvidia", vram_bytes=8 * 1024**3),
|
|
GPUInfo(name="GPU 1", vendor="intel", vram_bytes=0, shared_memory=True),
|
|
],
|
|
cpu_name="CPU",
|
|
cpu_cores=8,
|
|
ram_bytes=32 * 1024**3,
|
|
disk_free_bytes=100 * 1024**3,
|
|
os="linux",
|
|
)
|
|
|
|
with pytest.raises(Exit):
|
|
_apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=4.0)
|
|
|
|
|
|
def test_apply_gpu_overrides_updates_simulated_gpu_bandwidth():
|
|
hw = HardwareInfo(gpus=[], ram_bytes=32 * 1024**3, os="linux")
|
|
|
|
_apply_gpu_overrides(
|
|
hw, cpu_only=False, gpu=["Unknown GPU"], vram=4.0, bandwidth=72.0
|
|
)
|
|
|
|
assert len(hw.gpus) == 1
|
|
assert hw.gpus[0].vram_bytes == 4 * 1024**3
|
|
assert hw.gpus[0].memory_bandwidth_gbps == 72.0
|
|
|
|
|
|
def test_apply_gpu_overrides_rejects_override_without_gpu():
|
|
hw = HardwareInfo(gpus=[], ram_bytes=32 * 1024**3, os="linux")
|
|
|
|
with pytest.raises(Exit):
|
|
_apply_gpu_overrides(hw, cpu_only=False, gpu=None, vram=4.0)
|
|
|
|
|
|
def test_include_vision_candidates_by_profile():
|
|
assert _include_vision_candidates("vision") is True
|
|
assert _include_vision_candidates("any") is True
|
|
assert _include_vision_candidates("general") is False
|
|
assert _include_vision_candidates("coding") is False
|
|
|
|
|
|
def test_fill_missing_published_at_updates_models():
|
|
model = ModelInfo(
|
|
id="Qwen/Qwen3-8B-AWQ",
|
|
family_id="qwen3-8b",
|
|
name="Qwen3-8B-AWQ",
|
|
parameter_count=8_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
)
|
|
result = CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=None,
|
|
can_run=True,
|
|
vram_required_bytes=0,
|
|
vram_available_bytes=0,
|
|
)
|
|
|
|
async def _fake_fetch(ids: list[str]) -> dict[str, str]:
|
|
assert ids == ["Qwen/Qwen3-8B-AWQ"]
|
|
return {"Qwen/Qwen3-8B-AWQ": "2026-03-05T08:00:00.000Z"}
|
|
|
|
updated = _fill_missing_published_at([model], [result], _fake_fetch)
|
|
assert updated is True
|
|
assert model.published_at == "2026-03-05T08:00:00.000Z"
|
|
|
|
|
|
def test_version_option_prints_version_and_exits():
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["--version"])
|
|
assert result.exit_code == 0
|
|
assert _current_version() in result.stdout
|
|
|
|
|
|
def test_module_entrypoint_uses_cli_app():
|
|
assert main_mod.app is app
|
|
|
|
|
|
def test_format_fetch_error_uses_exception_class_when_message_is_empty():
|
|
class EmptyNetworkError(Exception):
|
|
def __str__(self) -> str:
|
|
return ""
|
|
|
|
assert _format_fetch_error(EmptyNetworkError()) == (
|
|
"EmptyNetworkError with no detail from the network layer"
|
|
)
|
|
|
|
|
|
def test_format_fetch_error_includes_status_and_url_for_empty_http_error():
|
|
request = httpx.Request("GET", "https://huggingface.co/api/models")
|
|
response = httpx.Response(429, request=request)
|
|
error = httpx.HTTPStatusError("", request=request, response=response)
|
|
|
|
assert _format_fetch_error(error) == (
|
|
"HTTPStatusError: HTTP 429 for https://huggingface.co/api/models"
|
|
)
|
|
|
|
|
|
def test_merge_model_eval_benchmarks_is_now_a_noop():
|
|
"""As of the self_reported evidence tier, _merge_model_eval_benchmarks
|
|
must NOT mutate the leaderboard scores. Uploader-reported hf_eval values
|
|
are consumed directly by the ranker as a separate, low-trust source.
|
|
"""
|
|
model_direct_missing = ModelInfo(
|
|
id="meta-llama/Llama-3.1-8B-Instruct",
|
|
family_id="llama-3.1-8b",
|
|
name="Llama-3.1-8B-Instruct",
|
|
parameter_count=8_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
benchmark_scores={"hf_eval": 66.4},
|
|
)
|
|
model_already_present = ModelInfo(
|
|
id="Qwen/Qwen2.5-7B-Instruct",
|
|
family_id="qwen2.5-7b",
|
|
name="Qwen2.5-7B-Instruct",
|
|
parameter_count=7_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
benchmark_scores={"hf_eval": 70.0},
|
|
)
|
|
original = {"Qwen/Qwen2.5-7B-Instruct": 71.2}
|
|
merged, injected = _merge_model_eval_benchmarks(
|
|
[model_direct_missing, model_already_present],
|
|
original,
|
|
)
|
|
# Function is a deprecation no-op now.
|
|
assert injected == 0
|
|
assert merged is original or merged == original
|
|
# Critically, the uploader-reported value MUST NOT have been injected
|
|
# under the model id, because doing so would make it appear as a
|
|
# direct leaderboard hit.
|
|
assert "meta-llama/Llama-3.1-8B-Instruct" not in merged
|
|
|
|
|
|
def test_validate_evidence_accepts_all_modes():
|
|
assert _validate_evidence("strict") == "strict"
|
|
assert _validate_evidence("base") == "base"
|
|
assert _validate_evidence("any") == "any"
|
|
|
|
|
|
def test_validate_evidence_rejects_unknown_mode():
|
|
with pytest.raises(Exit):
|
|
_validate_evidence("foo")
|
|
|
|
|
|
def test_resolve_evidence_mode_direct_alias_wins():
|
|
assert _resolve_evidence_mode("base", direct=True) == "strict"
|
|
|
|
|
|
def test_resolve_fit_filter_accepts_gpu_only_alias():
|
|
assert _resolve_fit_filter("any", gpu_only=False) == "any"
|
|
assert _resolve_fit_filter("gpu", gpu_only=False) == "full_gpu"
|
|
assert _resolve_fit_filter("full-gpu", gpu_only=False) == "full_gpu"
|
|
assert _resolve_fit_filter("full_gpu", gpu_only=False) == "full_gpu"
|
|
assert _resolve_fit_filter("any", gpu_only=True) == "full_gpu"
|
|
|
|
|
|
def test_resolve_fit_filter_rejects_unknown_mode():
|
|
with pytest.raises(Exit):
|
|
_resolve_fit_filter("partial", gpu_only=False)
|
|
|
|
|
|
def test_resolve_speed_filter_presets_and_min_speed_override():
|
|
assert _resolve_speed_filter("any", min_speed=None) is None
|
|
assert _resolve_speed_filter("usable", min_speed=None) == 10.0
|
|
assert _resolve_speed_filter("fast", min_speed=None) == 30.0
|
|
assert _resolve_speed_filter("fast", min_speed=2.5) == 2.5
|
|
|
|
|
|
def test_resolve_speed_filter_rejects_unknown_mode():
|
|
with pytest.raises(Exit):
|
|
_resolve_speed_filter("slowish", min_speed=None)
|
|
|
|
|
|
def test_parse_memory_amount_supports_gb_mb_and_percent():
|
|
assert _parse_memory_amount("1.5GB", option_name="--x") == int(1.5 * 1024**3)
|
|
assert _parse_memory_amount("512MB", option_name="--x") == 512 * 1024**2
|
|
assert _parse_memory_amount("8", option_name="--x") == 8 * 1024**3
|
|
assert (
|
|
_parse_memory_amount("10%", option_name="--x", total_bytes=20 * 1024**3)
|
|
== 2 * 1024**3
|
|
)
|
|
|
|
|
|
def test_apply_memory_budgets_sets_vram_headroom_and_ram_budget():
|
|
hw = _hw_with_gpu(16)
|
|
|
|
_apply_memory_budgets(hw, vram_headroom="1GB", ram_budget="8GB")
|
|
|
|
assert hw.gpus[0].vram_bytes == 16 * 1024**3
|
|
assert hw.gpus[0].usable_vram_bytes == 15 * 1024**3
|
|
assert hw.ram_budget_bytes == 8 * 1024**3
|
|
assert any("VRAM headroom" in note for note in hw.budget_notes)
|
|
assert any("RAM budget" in note for note in hw.budget_notes)
|
|
|
|
|
|
def test_apply_memory_budgets_validates_vram_headroom_without_gpus():
|
|
hw = HardwareInfo(gpus=[], ram_bytes=16 * 1024**3)
|
|
|
|
with pytest.raises(Exit):
|
|
_apply_memory_budgets(hw, vram_headroom="nope", ram_budget=None)
|
|
|
|
|
|
def test_apply_memory_budgets_accepts_valid_noop_vram_headroom_without_gpus():
|
|
hw = HardwareInfo(gpus=[], ram_bytes=16 * 1024**3)
|
|
|
|
_apply_memory_budgets(hw, vram_headroom="10%", ram_budget=None)
|
|
|
|
assert hw.gpus == []
|
|
assert hw.ram_budget_bytes is None
|
|
|
|
|
|
def test_main_passes_gpu_only_fit_filter(monkeypatch):
|
|
model = ModelInfo(
|
|
id="org/Test-7B",
|
|
family_id="test-7b",
|
|
name="Test-7B",
|
|
parameter_count=7_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
published_at="2026-01-01T00:00:00.000Z",
|
|
)
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
captured["fit_filter"] = kwargs.get("fit_filter")
|
|
return [
|
|
CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=None,
|
|
can_run=True,
|
|
vram_required_bytes=4 * 1024**3,
|
|
vram_available_bytes=8 * 1024**3,
|
|
fit_type="full_gpu",
|
|
quality_score=80.0,
|
|
)
|
|
]
|
|
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_hardware", lambda hardware: None
|
|
)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_ranking",
|
|
lambda results, **kwargs: None,
|
|
)
|
|
|
|
result = CliRunner().invoke(app, ["--gpu-only"])
|
|
|
|
assert result.exit_code == 0
|
|
assert captured["fit_filter"] == "full_gpu"
|
|
|
|
|
|
def test_main_passes_speed_preset_and_default_runtime_columns(monkeypatch):
|
|
model = ModelInfo(
|
|
id="org/Test-7B",
|
|
family_id="test-7b",
|
|
name="Test-7B",
|
|
parameter_count=7_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
published_at="2026-01-01T00:00:00.000Z",
|
|
)
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
captured["min_speed"] = kwargs.get("min_speed")
|
|
return [
|
|
CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=None,
|
|
can_run=True,
|
|
vram_required_bytes=4 * 1024**3,
|
|
vram_available_bytes=8 * 1024**3,
|
|
fit_type="full_gpu",
|
|
estimated_tok_per_sec=8.0,
|
|
quality_score=80.0,
|
|
)
|
|
]
|
|
|
|
def fake_display_ranking(results, **kwargs):
|
|
captured["show_status"] = kwargs.get("show_status")
|
|
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_hardware", lambda hardware: None
|
|
)
|
|
monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking)
|
|
|
|
result = CliRunner().invoke(app, ["--speed", "usable"])
|
|
|
|
assert result.exit_code == 0
|
|
assert captured["min_speed"] == 10.0
|
|
assert captured["show_status"] is True
|
|
|
|
|
|
def test_main_details_flag_restores_metadata_columns(monkeypatch):
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
return []
|
|
|
|
def fake_display_ranking(results, **kwargs):
|
|
captured["show_status"] = kwargs.get("show_status")
|
|
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_hardware", lambda hardware: None
|
|
)
|
|
monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking)
|
|
|
|
result = CliRunner().invoke(app, ["--details", "--min-params", "1"])
|
|
|
|
assert result.exit_code == 0
|
|
assert captured["show_status"] is False
|
|
|
|
|
|
def test_main_markdown_alias_dispatches_markdown_output(monkeypatch):
|
|
model = ModelInfo(
|
|
id="org/Test-7B",
|
|
family_id="test-7b",
|
|
name="Test-7B",
|
|
parameter_count=7_000_000_000,
|
|
downloads=1,
|
|
likes=1,
|
|
published_at="2026-01-01T00:00:00.000Z",
|
|
)
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
return [
|
|
CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=None,
|
|
can_run=True,
|
|
vram_required_bytes=4 * 1024**3,
|
|
vram_available_bytes=8 * 1024**3,
|
|
fit_type="full_gpu",
|
|
estimated_tok_per_sec=12.0,
|
|
quality_score=80.0,
|
|
)
|
|
]
|
|
|
|
def fake_display_markdown(results, hardware, **kwargs):
|
|
captured["called"] = True
|
|
captured["show_status"] = kwargs.get("show_status")
|
|
|
|
def fail_display_hardware(hardware):
|
|
raise AssertionError("markdown output should not render Rich hardware panel")
|
|
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_markdown", fake_display_markdown
|
|
)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_hardware", fail_display_hardware
|
|
)
|
|
|
|
result = CliRunner().invoke(app, ["-m"])
|
|
|
|
assert result.exit_code == 0
|
|
assert captured["called"] is True
|
|
assert captured["show_status"] is True
|
|
|
|
|
|
def test_main_json_and_markdown_are_mutually_exclusive():
|
|
result = CliRunner().invoke(app, ["--json", "--markdown"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--json and --markdown are mutually exclusive" in result.stdout
|
|
|
|
|
|
def test_main_top_zero_rejected():
|
|
result = CliRunner().invoke(app, ["--top", "0"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--top must be 1 or greater" in result.stdout
|
|
|
|
|
|
def test_main_top_negative_rejected():
|
|
# ``results[:-5]`` would silently drop the best-ranked models.
|
|
result = CliRunner().invoke(app, ["--top=-5"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--top must be 1 or greater" in result.stdout
|
|
|
|
|
|
def test_main_negative_min_speed_rejected():
|
|
result = CliRunner().invoke(app, ["--min-speed=-1"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--min-speed must be 0 or greater" in result.stdout
|
|
|
|
|
|
def test_main_negative_min_params_rejected():
|
|
result = CliRunner().invoke(app, ["--min-params=-2"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--min-params must be 0 or greater" in result.stdout
|
|
|
|
|
|
def test_validate_ranking_flags_accepts_valid_values():
|
|
# A valid combination must not raise (no recommendations are dropped).
|
|
cli_mod._validate_ranking_flags(top=10, min_speed=0.0, min_params=None)
|
|
cli_mod._validate_ranking_flags(top=1, min_speed=None, min_params=7.0)
|
|
|
|
|
|
def test_upgrade_top_zero_rejected():
|
|
# The upgrade command shares the same --top -> results[:top_n] hazard.
|
|
result = CliRunner().invoke(app, ["upgrade", "RTX 4090", "--top", "0"])
|
|
|
|
assert result.exit_code == 1
|
|
assert "--top must be 1 or greater" in result.stdout
|
|
|
|
|
|
def test_main_empty_gpu_only_result_shows_fit_message(monkeypatch):
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
return []
|
|
|
|
def fake_display_ranking(results, **kwargs):
|
|
captured["empty_message"] = kwargs.get("empty_message")
|
|
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(
|
|
"whichllm.output.display.display_hardware", lambda hardware: None
|
|
)
|
|
monkeypatch.setattr("whichllm.output.display.display_ranking", fake_display_ranking)
|
|
|
|
result = CliRunner().invoke(app, ["--fit", "full-gpu", "--min-params", "1"])
|
|
|
|
assert result.exit_code == 0
|
|
assert "No full-GPU models found" in captured["empty_message"]
|
|
|
|
|
|
# --------------- plan command tests ---------------
|
|
|
|
|
|
def test_plan_no_model_found_shows_error(monkeypatch):
|
|
monkeypatch.setattr("whichllm.models.cache.load_cache", lambda: [])
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["plan", "nonexistent_model_xyz_999"])
|
|
assert result.exit_code != 0
|
|
assert "No model found" in result.stdout
|
|
|
|
|
|
def test_plan_display_plan_renders_tables():
|
|
"""display_plan should render model info, VRAM table, and GPU table."""
|
|
from whichllm.output.display import display_plan
|
|
|
|
model = ModelInfo(
|
|
id="test-org/Test-Model-7B-GGUF",
|
|
family_id="test-7b",
|
|
name="Test-Model-7B",
|
|
parameter_count=7_000_000_000,
|
|
architecture="llama",
|
|
context_length=4096,
|
|
license="mit",
|
|
downloads=100,
|
|
likes=10,
|
|
)
|
|
# Should not raise
|
|
display_plan(model, context_length=4096, target_quant="Q4_K_M")
|
|
|
|
|
|
def test_plan_display_plan_json_outputs_valid_json():
|
|
"""display_plan_json should output valid JSON."""
|
|
import json as json_mod
|
|
from io import StringIO
|
|
|
|
from rich.console import Console
|
|
|
|
from whichllm.output.display import display_plan_json
|
|
|
|
model = ModelInfo(
|
|
id="test-org/Test-Model-7B-GGUF",
|
|
family_id="test-7b",
|
|
name="Test-Model-7B",
|
|
parameter_count=7_000_000_000,
|
|
architecture="llama",
|
|
context_length=4096,
|
|
license="mit",
|
|
downloads=100,
|
|
likes=10,
|
|
)
|
|
# Capture output
|
|
buf = StringIO()
|
|
import whichllm.output._console as console_mod
|
|
|
|
orig_console = console_mod.console
|
|
console_mod.console = Console(file=buf, force_terminal=False)
|
|
try:
|
|
display_plan_json(model, context_length=4096, target_quant="Q4_K_M")
|
|
finally:
|
|
console_mod.console = orig_console
|
|
raw = buf.getvalue().strip()
|
|
data = json_mod.loads(raw)
|
|
assert data["model"]["id"] == "test-org/Test-Model-7B-GGUF"
|
|
assert "vram_by_quant" in data
|
|
assert "gpu_compatibility" in data
|
|
assert data["target_quant"] == "Q4_K_M"
|
|
|
|
|
|
# --------------- helper tests ---------------
|
|
|
|
|
|
def _make_model(
|
|
model_id="org/Test-7B-GGUF",
|
|
downloads=100,
|
|
gguf_variants=None,
|
|
parameter_count=7_000_000_000,
|
|
):
|
|
return ModelInfo(
|
|
id=model_id,
|
|
family_id="test-7b",
|
|
name="Test-7B",
|
|
parameter_count=parameter_count,
|
|
downloads=downloads,
|
|
likes=10,
|
|
gguf_variants=gguf_variants or [],
|
|
)
|
|
|
|
|
|
def test_search_model_exact_match():
|
|
models = [_make_model("org/Llama-8B"), _make_model("org/Qwen-7B")]
|
|
result = _search_model(models, "org/Llama-8B")
|
|
assert result.id == "org/Llama-8B"
|
|
|
|
|
|
def test_search_model_endswith_match():
|
|
models = [_make_model("org/Llama-8B"), _make_model("org/Qwen-7B")]
|
|
result = _search_model(models, "Llama-8B")
|
|
assert result.id == "org/Llama-8B"
|
|
|
|
|
|
def test_search_model_term_match():
|
|
models = [_make_model("org/Llama-3.1-8B-GGUF"), _make_model("org/Qwen-7B")]
|
|
result = _search_model(models, "llama 8b")
|
|
assert result.id == "org/Llama-3.1-8B-GGUF"
|
|
|
|
|
|
def test_search_model_not_found():
|
|
models = [_make_model("org/Llama-8B")]
|
|
with pytest.raises(Exit):
|
|
_search_model(models, "nonexistent_xyz")
|
|
|
|
|
|
# --- regression tests for size-token substring matching (#107) ---
|
|
|
|
|
|
def test_search_model_7b_does_not_match_1_7b():
|
|
"""'qwen 7b' should NOT match Qwen3-1.7B (issue #107)."""
|
|
models = [
|
|
_make_model(
|
|
"org/Qwen3-1.7B-GGUF", downloads=9999, parameter_count=1_700_000_000
|
|
),
|
|
_make_model("org/Qwen3-7B-GGUF", downloads=100, parameter_count=7_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 7b")
|
|
assert result.id == "org/Qwen3-7B-GGUF"
|
|
|
|
|
|
def test_search_model_2b_does_not_match_12b():
|
|
"""'gemma 2b' should NOT match gemma-3-12b-it (issue #107)."""
|
|
models = [
|
|
_make_model(
|
|
"google/gemma-3-12b-it", downloads=5000, parameter_count=12_000_000_000
|
|
),
|
|
_make_model("google/gemma-2b", downloads=100, parameter_count=2_000_000_000),
|
|
]
|
|
result = _search_model(models, "gemma 2b")
|
|
assert result.id == "google/gemma-2b"
|
|
|
|
|
|
def test_search_model_3b_does_not_match_30b_a3b():
|
|
"""'qwen 3b' should NOT match Qwen3-30B-A3B (issue #107)."""
|
|
models = [
|
|
_make_model(
|
|
"org/Qwen3-30B-A3B-GGUF", downloads=8000, parameter_count=30_000_000_000
|
|
),
|
|
_make_model("org/Qwen3-3B-GGUF", downloads=50, parameter_count=3_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 3b")
|
|
assert result.id == "org/Qwen3-3B-GGUF"
|
|
|
|
|
|
def test_search_model_no_size_token_still_works():
|
|
"""Queries without a size token should still use plain substring matching."""
|
|
models = [
|
|
_make_model("org/Llama-3.1-8B-GGUF", parameter_count=8_000_000_000),
|
|
_make_model("org/Qwen-7B", parameter_count=7_000_000_000),
|
|
]
|
|
result = _search_model(models, "llama gguf")
|
|
assert result.id == "org/Llama-3.1-8B-GGUF"
|
|
|
|
|
|
def test_search_model_millions_size_token():
|
|
"""'500m' size token should match a ~500M parameter model."""
|
|
models = [
|
|
_make_model("org/SmolLM-500M", downloads=200, parameter_count=500_000_000),
|
|
_make_model("org/SmolLM-1.7B", downloads=300, parameter_count=1_700_000_000),
|
|
]
|
|
result = _search_model(models, "smollm 500m")
|
|
assert result.id == "org/SmolLM-500M"
|
|
|
|
|
|
def test_search_model_decimal_size_token():
|
|
"""'0.5b' size token should match a ~500M parameter model."""
|
|
models = [
|
|
_make_model("org/TinyModel-0.5B", downloads=100, parameter_count=500_000_000),
|
|
_make_model("org/TinyModel-3B", downloads=500, parameter_count=3_000_000_000),
|
|
]
|
|
result = _search_model(models, "tinymodel 0.5b")
|
|
assert result.id == "org/TinyModel-0.5B"
|
|
|
|
|
|
def test_search_model_zero_param_count_passes_through():
|
|
"""Models with parameter_count=0 (missing metadata) should not be excluded."""
|
|
models = [
|
|
_make_model("org/Mystery-7B-GGUF", downloads=500, parameter_count=0),
|
|
]
|
|
result = _search_model(models, "mystery 7b")
|
|
assert result.id == "org/Mystery-7B-GGUF"
|
|
|
|
|
|
def test_search_model_first_size_token_wins():
|
|
"""When multiple size tokens appear, only the first is used as a filter."""
|
|
models = [
|
|
_make_model(
|
|
"org/Qwen3-30B-A3B-GGUF", downloads=8000, parameter_count=30_000_000_000
|
|
),
|
|
_make_model(
|
|
"org/Qwen3-7B-3B-GGUF", downloads=100, parameter_count=7_000_000_000
|
|
),
|
|
]
|
|
# "7b" is the first size token and filters by ~7B; "3b" becomes a text term
|
|
result = _search_model(models, "qwen 7b 3b")
|
|
assert result.id == "org/Qwen3-7B-3B-GGUF"
|
|
|
|
|
|
def test_search_model_7b_prefers_closest_size_over_downloads():
|
|
"""'qwen 7b' should pick 8B (closest to 7B) over 4B even if 4B has more downloads."""
|
|
models = [
|
|
_make_model("org/Qwen3-4B-GGUF", downloads=9000, parameter_count=4_000_000_000),
|
|
_make_model("org/Qwen3-8B-GGUF", downloads=100, parameter_count=8_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 7b")
|
|
assert result.id == "org/Qwen3-8B-GGUF"
|
|
|
|
|
|
def test_search_model_3b_prefers_closest_size_over_downloads():
|
|
"""'qwen 3b' should pick 3B (exact) over 4B even if 4B has more downloads."""
|
|
models = [
|
|
_make_model("org/Qwen3-4B-GGUF", downloads=9000, parameter_count=4_000_000_000),
|
|
_make_model("org/Qwen3-3B-GGUF", downloads=50, parameter_count=3_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 3b")
|
|
assert result.id == "org/Qwen3-3B-GGUF"
|
|
|
|
|
|
def test_search_model_size_tiebreak_falls_back_to_downloads():
|
|
"""When two models are equally close in size, prefer the one with more downloads."""
|
|
models = [
|
|
_make_model(
|
|
"org/Qwen3-8B-A-GGUF", downloads=100, parameter_count=8_000_000_000
|
|
),
|
|
_make_model(
|
|
"org/Qwen3-8B-B-GGUF", downloads=500, parameter_count=8_000_000_000
|
|
),
|
|
]
|
|
result = _search_model(models, "qwen 7b")
|
|
assert result.id == "org/Qwen3-8B-B-GGUF"
|
|
|
|
|
|
def test_search_model_unknown_param_count_ranks_after_known():
|
|
"""Models with parameter_count=0 should rank after models with known sizes."""
|
|
models = [
|
|
_make_model("org/Qwen3-Unknown-GGUF", downloads=9999, parameter_count=0),
|
|
_make_model("org/Qwen3-7B-GGUF", downloads=10, parameter_count=7_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 7b")
|
|
assert result.id == "org/Qwen3-7B-GGUF"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_id, expected",
|
|
[
|
|
("org/Qwen3-8B-GGUF", 8.0),
|
|
("org/Qwen3.5-9B-NVFP4", 9.0),
|
|
("org/Qwen3-30B-A3B-GGUF", 30.0),
|
|
("org/SmolLM-500M", 0.5),
|
|
("org/TinyModel-1.7B-Chat", 1.7),
|
|
("org/Llama-3.1-8B-Instruct", 8.0),
|
|
("org/NoSizeLabel-GGUF", None),
|
|
],
|
|
)
|
|
def test_extract_id_size_b(model_id, expected):
|
|
assert _extract_id_size_b(model_id) == expected
|
|
|
|
|
|
def test_search_model_7b_prefers_id_label_over_param_count():
|
|
"""'qwen 7b' should not pick a 9B-labeled repo even if its param count is closer."""
|
|
models = [
|
|
_make_model(
|
|
"org/Qwen3.5-9B-NVFP4", downloads=500, parameter_count=6_600_000_000
|
|
),
|
|
_make_model("org/Qwen3-8B-GGUF", downloads=100, parameter_count=8_000_000_000),
|
|
]
|
|
result = _search_model(models, "qwen 7b")
|
|
assert result.id == "org/Qwen3-8B-GGUF"
|
|
|
|
|
|
def test_pick_gguf_variant_by_preference():
|
|
variants = [
|
|
GGUFVariant(filename="q2.gguf", quant_type="Q2_K", file_size_bytes=1000),
|
|
GGUFVariant(filename="q4km.gguf", quant_type="Q4_K_M", file_size_bytes=2000),
|
|
]
|
|
model = _make_model(gguf_variants=variants)
|
|
result = _pick_gguf_variant(model)
|
|
assert result.quant_type == "Q4_K_M"
|
|
|
|
|
|
def test_pick_gguf_variant_with_filter():
|
|
variants = [
|
|
GGUFVariant(filename="q2.gguf", quant_type="Q2_K", file_size_bytes=1000),
|
|
GGUFVariant(filename="q4km.gguf", quant_type="Q4_K_M", file_size_bytes=2000),
|
|
]
|
|
model = _make_model(gguf_variants=variants)
|
|
result = _pick_gguf_variant(model, quant_filter="Q2_K")
|
|
assert result.quant_type == "Q2_K"
|
|
|
|
|
|
def test_pick_gguf_variant_no_variants():
|
|
model = _make_model(gguf_variants=[])
|
|
result = _pick_gguf_variant(model)
|
|
assert result is None
|
|
|
|
|
|
def test_resolve_ranked_synthetic_gguf_to_real_repo():
|
|
selected = ModelInfo(
|
|
id="Qwen/Qwen3.6-27B",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B",
|
|
parameter_count=27_000_000_000,
|
|
downloads=50_000,
|
|
)
|
|
real_gguf = ModelInfo(
|
|
id="unsloth/Qwen3.6-27B-GGUF",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B-GGUF",
|
|
parameter_count=27_000_000_000,
|
|
downloads=200_000,
|
|
base_model="Qwen/Qwen3.6-27B",
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="Qwen3.6-27B-Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="Qwen3.6-27B.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
|
|
resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, real_gguf])
|
|
|
|
assert resolved is not None
|
|
model, variant = resolved
|
|
assert model.id == "unsloth/Qwen3.6-27B-GGUF"
|
|
assert variant.filename == "Qwen3.6-27B-Q4_K_M.gguf"
|
|
|
|
|
|
def test_resolve_ranked_synthetic_gguf_prefers_exact_quant():
|
|
selected = ModelInfo(
|
|
id="Qwen/Qwen3.6-27B",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B",
|
|
parameter_count=27_000_000_000,
|
|
)
|
|
q5_only = ModelInfo(
|
|
id="converter/Qwen3.6-27B-GGUF",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B-GGUF",
|
|
parameter_count=27_000_000_000,
|
|
downloads=1_000_000,
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="q5.gguf",
|
|
quant_type="Q5_K_M",
|
|
file_size_bytes=18_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
q4_match = ModelInfo(
|
|
id="smaller/Qwen3.6-27B-GGUF",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B-GGUF",
|
|
parameter_count=27_000_000_000,
|
|
downloads=10,
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="q4.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="Qwen3.6-27B.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
|
|
resolved = _resolve_ranked_gguf_for_run(
|
|
selected,
|
|
synthetic,
|
|
[selected, q5_only, q4_match],
|
|
)
|
|
|
|
assert resolved is not None
|
|
model, variant = resolved
|
|
assert model.id == "smaller/Qwen3.6-27B-GGUF"
|
|
assert variant.quant_type == "Q4_K_M"
|
|
|
|
|
|
def test_resolve_ranked_synthetic_gguf_rejects_quant_mismatch():
|
|
selected = ModelInfo(
|
|
id="Qwen/Qwen3.6-27B",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B",
|
|
parameter_count=27_000_000_000,
|
|
)
|
|
q5_only = ModelInfo(
|
|
id="converter/Qwen3.6-27B-GGUF",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B-GGUF",
|
|
parameter_count=27_000_000_000,
|
|
downloads=1_000_000,
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="q5.gguf",
|
|
quant_type="Q5_K_M",
|
|
file_size_bytes=18_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="Qwen3.6-27B.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
|
|
resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, q5_only])
|
|
|
|
assert resolved is None
|
|
|
|
|
|
def test_resolve_ranked_synthetic_gguf_without_real_repo_returns_none():
|
|
selected = ModelInfo(
|
|
id="Qwen/Qwen3.6-27B",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B",
|
|
parameter_count=27_000_000_000,
|
|
)
|
|
unrelated = ModelInfo(
|
|
id="other/Model-7B-GGUF",
|
|
family_id="model-7b",
|
|
name="Model-7B-GGUF",
|
|
parameter_count=7_000_000_000,
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="other.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=4_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="Qwen3.6-27B.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
|
|
assert (
|
|
_resolve_ranked_gguf_for_run(selected, synthetic, [selected, unrelated]) is None
|
|
)
|
|
|
|
|
|
def test_resolve_ranked_synthetic_gguf_rejects_size_mismatch():
|
|
selected = ModelInfo(
|
|
id="deepseek-ai/DeepSeek-V4-Flash",
|
|
family_id="deepseek-v4-flash",
|
|
name="DeepSeek-V4-Flash",
|
|
parameter_count=158_000_000_000,
|
|
)
|
|
mtp_head = ModelInfo(
|
|
id="converter/deepseek-v4-flash-mtp-gguf",
|
|
family_id="deepseek-v4-flash",
|
|
name="DeepSeek-V4-Flash-MTP-GGUF",
|
|
parameter_count=6_600_000_000,
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="mtp.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=4_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="DeepSeek-V4-Flash.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=90_000_000_000,
|
|
)
|
|
|
|
resolved = _resolve_ranked_gguf_for_run(selected, synthetic, [selected, mtp_head])
|
|
|
|
assert resolved is None
|
|
|
|
|
|
# --------------- run/snippet command tests ---------------
|
|
|
|
|
|
def test_run_exits_gracefully():
|
|
"""run should fail gracefully (uv missing, or no model found)."""
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["run", "some-model"])
|
|
if result.exit_code != 0:
|
|
assert any(
|
|
msg in result.stdout
|
|
for msg in ("uv is required", "No model found", "llama-cpp-python")
|
|
)
|
|
|
|
|
|
def test_transformers_chat_script_passes_tokenizer_mapping_to_generate():
|
|
model = _make_model(model_id="org/Test-7B")
|
|
|
|
script = _generate_chat_script(
|
|
model, variant=None, context_length=4096, cpu_only=False
|
|
)
|
|
|
|
assert "return_dict=True" in script
|
|
assert "kwargs=dict(**inputs, max_new_tokens=512, streamer=streamer)" in script
|
|
assert "kwargs=dict(input_ids=inputs" not in script
|
|
|
|
|
|
def test_transformers_chat_script_provides_disk_offload_folder():
|
|
model = _make_model(model_id="org/Test-7B")
|
|
|
|
script = _generate_chat_script(
|
|
model, variant=None, context_length=4096, cpu_only=False
|
|
)
|
|
|
|
assert 'tempfile.mkdtemp(prefix="whichllm_transformers_offload_")' in script
|
|
assert "offload_folder=offload_folder" in script
|
|
assert "shutil.rmtree(offload_folder, ignore_errors=True)" in script
|
|
|
|
|
|
def test_run_auto_pick_resolves_ranked_gguf_before_launch(monkeypatch):
|
|
selected = ModelInfo(
|
|
id="Qwen/Qwen3.6-27B",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B",
|
|
parameter_count=27_000_000_000,
|
|
downloads=50_000,
|
|
)
|
|
real_gguf = ModelInfo(
|
|
id="unsloth/Qwen3.6-27B-GGUF",
|
|
family_id="qwen3-27b",
|
|
name="Qwen3.6-27B-GGUF",
|
|
parameter_count=27_000_000_000,
|
|
downloads=200_000,
|
|
base_model="Qwen/Qwen3.6-27B",
|
|
gguf_variants=[
|
|
GGUFVariant(
|
|
filename="q4.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
],
|
|
)
|
|
synthetic = GGUFVariant(
|
|
filename="Qwen3.6-27B.Q4_K_M.gguf",
|
|
quant_type="Q4_K_M",
|
|
file_size_bytes=16_000_000_000,
|
|
)
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_rank_models(models, hardware, **kwargs):
|
|
captured["quant_filter"] = kwargs.get("quant_filter")
|
|
return [
|
|
CompatibilityResult(
|
|
model=selected,
|
|
gguf_variant=synthetic,
|
|
can_run=True,
|
|
vram_required_bytes=0,
|
|
vram_available_bytes=0,
|
|
quality_score=90.0,
|
|
)
|
|
]
|
|
|
|
def fake_generate_chat_script(model, variant, context_length, cpu_only):
|
|
captured["model_id"] = model.id
|
|
captured["variant"] = variant
|
|
return "print('ok')"
|
|
|
|
class Completed:
|
|
returncode = 0
|
|
|
|
def fake_run(cmd):
|
|
captured["cmd"] = cmd
|
|
return Completed()
|
|
|
|
monkeypatch.setattr("shutil.which", lambda _: "/usr/bin/uv")
|
|
monkeypatch.setattr(cli_mod, "_load_models", lambda refresh: [selected, real_gguf])
|
|
monkeypatch.setattr(
|
|
"whichllm.hardware.detector.detect_hardware", lambda: _hw_with_gpu(8)
|
|
)
|
|
monkeypatch.setattr("whichllm.models.benchmark.load_benchmark_cache", lambda: {})
|
|
monkeypatch.setattr("whichllm.engine.ranker.rank_models", fake_rank_models)
|
|
monkeypatch.setattr(cli_mod, "_generate_chat_script", fake_generate_chat_script)
|
|
monkeypatch.setattr("subprocess.run", fake_run)
|
|
|
|
result = CliRunner().invoke(app, ["run", "--quant", "Q4_K_M"])
|
|
|
|
assert result.exit_code == 0
|
|
assert captured["quant_filter"] == "Q4_K_M"
|
|
assert captured["model_id"] == "unsloth/Qwen3.6-27B-GGUF"
|
|
assert captured["variant"].filename == "q4.gguf"
|
|
assert "llama-cpp-python" in captured["cmd"]
|
|
assert "transformers" not in captured["cmd"]
|
|
|
|
|
|
def test_snippet_no_model_found(monkeypatch):
|
|
monkeypatch.setattr(cli_mod, "_load_models", lambda refresh: [])
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["snippet", "nonexistent_model_xyz_999"])
|
|
assert result.exit_code != 0
|
|
assert "No model found" in result.stdout
|
|
|
|
|
|
def test_json_output_includes_benchmark_source_and_confidence():
|
|
"""display_json should include benchmark_source and benchmark_confidence."""
|
|
import json as json_mod
|
|
from io import StringIO
|
|
|
|
from rich.console import Console
|
|
|
|
from whichllm.output.display import display_json
|
|
|
|
model = ModelInfo(
|
|
id="test-org/Test-7B",
|
|
family_id="test-7b",
|
|
name="Test-7B",
|
|
parameter_count=7_000_000_000,
|
|
downloads=100,
|
|
likes=10,
|
|
)
|
|
result = CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=None,
|
|
can_run=True,
|
|
vram_required_bytes=8_000_000_000,
|
|
vram_available_bytes=24_000_000_000,
|
|
quality_score=55.0,
|
|
benchmark_status="estimated",
|
|
benchmark_source="line_interp",
|
|
benchmark_confidence=0.34,
|
|
)
|
|
hw = HardwareInfo(
|
|
gpus=[],
|
|
cpu_name="Test CPU",
|
|
cpu_cores=8,
|
|
ram_budget_bytes=32 * 1024**3,
|
|
ram_bytes=64 * 1024**3,
|
|
disk_free_bytes=500 * 1024**3,
|
|
os="linux",
|
|
budget_notes=["RAM budget: 32.0 GB"],
|
|
)
|
|
|
|
buf = StringIO()
|
|
import whichllm.output._console as console_mod
|
|
|
|
orig_console = console_mod.console
|
|
console_mod.console = Console(file=buf, force_terminal=False)
|
|
try:
|
|
display_json([result], hw)
|
|
finally:
|
|
console_mod.console = orig_console
|
|
|
|
data = json_mod.loads(buf.getvalue().strip())
|
|
entry = data["models"][0]
|
|
assert data["hardware"]["ram_budget_bytes"] == 32 * 1024**3
|
|
assert data["hardware"]["budget_notes"] == ["RAM budget: 32.0 GB"]
|
|
assert entry["artifact_repo_id"] is None
|
|
assert entry["artifact_filename"] is None
|
|
assert entry["benchmark_status"] == "estimated"
|
|
assert entry["benchmark_source"] == "line_interp"
|
|
assert entry["benchmark_confidence"] == 0.34
|
|
|
|
|
|
def test_json_output_includes_resolved_artifact_fields():
|
|
"""display_json should expose the actual GGUF repo/file when resolved."""
|
|
import json as json_mod
|
|
from io import StringIO
|
|
|
|
from rich.console import Console
|
|
|
|
from whichllm.output.display import display_json
|
|
|
|
model = ModelInfo(
|
|
id="Qwen/Qwen3-4B-Thinking-2507",
|
|
family_id="qwen3-4b-thinking",
|
|
name="Qwen3-4B-Thinking-2507",
|
|
parameter_count=4_000_000_000,
|
|
)
|
|
artifact = ModelInfo(
|
|
id="MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF",
|
|
family_id="qwen3-4b-thinking",
|
|
name="Qwen3-4B-Thinking-2507-GGUF",
|
|
parameter_count=4_000_000_000,
|
|
)
|
|
result = CompatibilityResult(
|
|
model=model,
|
|
gguf_variant=GGUFVariant(
|
|
filename="Qwen3-4B-Thinking-2507.Q3_K_M.gguf",
|
|
quant_type="Q3_K_M",
|
|
file_size_bytes=2_000_000_000,
|
|
),
|
|
artifact_model=artifact,
|
|
artifact_variant=GGUFVariant(
|
|
filename="Qwen3-4B-Thinking-2507-Q3_K_M.gguf",
|
|
quant_type="Q3_K_M",
|
|
file_size_bytes=2_000_000_000,
|
|
),
|
|
can_run=True,
|
|
vram_required_bytes=3_000_000_000,
|
|
vram_available_bytes=8_000_000_000,
|
|
)
|
|
hw = HardwareInfo(
|
|
gpus=[],
|
|
cpu_name="Test CPU",
|
|
cpu_cores=8,
|
|
ram_bytes=64 * 1024**3,
|
|
disk_free_bytes=500 * 1024**3,
|
|
os="linux",
|
|
)
|
|
|
|
buf = StringIO()
|
|
import whichllm.output._console as console_mod
|
|
|
|
orig_console = console_mod.console
|
|
console_mod.console = Console(file=buf, force_terminal=False)
|
|
try:
|
|
display_json([result], hw)
|
|
finally:
|
|
console_mod.console = orig_console
|
|
|
|
entry = json_mod.loads(buf.getvalue().strip())["models"][0]
|
|
assert entry["model_id"] == "Qwen/Qwen3-4B-Thinking-2507"
|
|
assert entry["artifact_repo_id"] == "MaziyarPanahi/Qwen3-4B-Thinking-2507-GGUF"
|
|
assert entry["artifact_filename"] == "Qwen3-4B-Thinking-2507-Q3_K_M.gguf"
|