# SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """ Path utilities for model and dataset handling """ import os import sys from pathlib import Path from typing import Optional import structlog from loggers import get_logger logger = get_logger(__name__) # Per-process cache to avoid repeated cache-dir scans for the same identifier. _CACHE_CASE_RESOLUTION_MEMO: dict[str, str] = {} # Instrumentation counters for operational visibility. _CACHE_CASE_RESOLUTION_STATS: dict[str, int] = { "calls": 0, "memo_hits": 0, "exact_hits": 0, "variant_hits": 0, "tie_breaks": 0, "fallbacks": 0, "errors": 0, } def _is_wsl() -> bool: """Detect if we are running inside WSL (Windows Subsystem for Linux).""" if sys.platform == "win32": return False try: with open("/proc/version", "r") as f: return "microsoft" in f.read().lower() except Exception: return False _IS_WSL: bool = _is_wsl() def normalize_path(path: str) -> str: """Normalize filesystem paths for cross-platform use. WSL maps drive-letter paths to ``/mnt//...``; native Windows keeps the drive and normalizes separators; elsewhere slashes are forward-only. """ if not path: return path # Handle Windows drive letters (C:\\ or c:\\) if len(path) >= 3 and path[1] == ":" and path[2] in ("\\", "/"): # Map to /mnt// only under WSL; native Windows keeps the drive letter. if _IS_WSL: drive = path[0].lower() rest = path[3:].replace("\\", "/") return f"/mnt/{drive}/{rest}" return path.replace("\\", "/") # Already Unix-style or relative return path.replace("\\", "/") def is_local_path(path: str) -> bool: """ Check if path is a local filesystem path vs HuggingFace model identifier. Examples: True: /home/user/model, C:\\models, ./model, ~/model False: unsloth/llama-3.1-8b, microsoft/phi-2 """ if not path: return False # Exists on disk → local (covers relative paths like "outputs/foo"). try: if Path(normalize_path(path)).expanduser().exists(): return True except Exception: pass # Obvious HF patterns if path.count("/") == 1 and not path.startswith(("/", ".", "~")): return False # Looks like org/model format # Filesystem indicators return ( path.startswith(("/", ".", "~")) # Unix absolute/relative or ":" in path # Windows drive or URL or "\\" in path # Windows separator or os.path.isabs(path) # System-absolute ) def get_cache_path(model_name: str) -> Optional[Path]: """Get HuggingFace cache path for a model if it exists.""" cache_dir = _hf_hub_cache_dir() resolved_name = resolve_cached_repo_id_case(model_name) model_cache_name = resolved_name.replace("/", "--") model_cache_path = cache_dir / f"models--{model_cache_name}" return model_cache_path if model_cache_path.exists() else None def is_model_cached(model_name: str) -> bool: """Check if model is downloaded in HuggingFace cache.""" cache_path = get_cache_path(model_name) if not cache_path: return False # Check for model files for suffix in [".safetensors", ".bin", ".json"]: if list(cache_path.rglob(f"*{suffix}")): return True return False def _hf_hub_cache_dir() -> Path: """Return HF cache root honoring HF_HUB_CACHE when available.""" try: from huggingface_hub.constants import HF_HUB_CACHE return Path(HF_HUB_CACHE) except Exception as exc: logger.debug( "Could not read huggingface_hub HF_HUB_CACHE, using default hub path: %s", exc, ) return Path.home() / ".cache" / "huggingface" / "hub" def resolve_cached_repo_id_case(model_name: str, use_memo: bool = True) -> str: """Resolve repo_id to the exact casing already present in local HF cache. Policy: prefer the requested/canonical repo_id, but reuse a case-variant's exact cached spelling if one already exists in local HF cache. Avoids duplicate downloads while preserving user intent where possible. """ _CACHE_CASE_RESOLUTION_STATS["calls"] += 1 if not model_name or "/" not in model_name: _CACHE_CASE_RESOLUTION_STATS["fallbacks"] += 1 return model_name cache_dir = _hf_hub_cache_dir() if not cache_dir.exists(): _CACHE_CASE_RESOLUTION_STATS["fallbacks"] += 1 return model_name expected_dir = f"models--{model_name.replace('/', '--')}" # Exact-case path first so a new exact match beats a memoized variant. exact_path = cache_dir / expected_dir if exact_path.is_dir(): if use_memo: _CACHE_CASE_RESOLUTION_MEMO[model_name] = model_name _CACHE_CASE_RESOLUTION_STATS["exact_hits"] += 1 return model_name # Revalidate memoized entries on disk to avoid stale results. if use_memo: cached = _CACHE_CASE_RESOLUTION_MEMO.get(model_name) if cached is not None: cached_path = cache_dir / f"models--{cached.replace('/', '--')}" if cached_path.is_dir(): _CACHE_CASE_RESOLUTION_STATS["memo_hits"] += 1 return cached # Stale entry -- drop it and re-scan below _CACHE_CASE_RESOLUTION_MEMO.pop(model_name, None) expected_lower = expected_dir.lower() try: candidates: list[str] = [] for entry in cache_dir.iterdir(): if not entry.is_dir(): continue if entry.name.lower() != expected_lower: continue if not entry.name.startswith("models--"): continue repo_part = entry.name[len("models--") :] if not repo_part: continue candidates.append(repo_part.replace("--", "/")) if candidates: # Deterministic tie-break if multiple case variants coexist resolved = sorted(candidates)[0] if len(candidates) > 1: _CACHE_CASE_RESOLUTION_STATS["tie_breaks"] += 1 _CACHE_CASE_RESOLUTION_STATS["variant_hits"] += 1 if use_memo: _CACHE_CASE_RESOLUTION_MEMO[model_name] = resolved return resolved except Exception as exc: _CACHE_CASE_RESOLUTION_STATS["errors"] += 1 logger.debug(f"Could not resolve cached repo_id case for '{model_name}': {exc}") _CACHE_CASE_RESOLUTION_STATS["fallbacks"] += 1 return model_name def get_cache_case_resolution_stats() -> dict[str, int]: """Return a copy of case-resolution instrumentation counters.""" return dict(_CACHE_CASE_RESOLUTION_STATS) def reset_cache_case_resolution_state() -> None: """Clear resolver memo and counters (primarily for tests).""" _CACHE_CASE_RESOLUTION_MEMO.clear() for key in _CACHE_CASE_RESOLUTION_STATS: _CACHE_CASE_RESOLUTION_STATS[key] = 0