import fs from 'node:fs' import path from 'node:path' import execa from 'execa' import { CPUArchitectures } from '@/types' import { LLAMACPP_BUILD_MANIFEST_PATH, CMAKE_BIN_PATH, LLAMACPP_BUILD_PATH, LLAMACPP_ROOT_MANIFEST_PATH, NINJA_BIN_PATH, LLAMACPP_SOURCE_BUILD_PATH, LLAMACPP_SOURCE_MANIFEST_PATH, LLAMACPP_SOURCE_PATH, LLAMACPP_PATH, LLAMACPP_RELEASE_VERSION } from '@/constants' import { FileHelper } from '@/helpers/file-helper' import { SystemHelper } from '@/helpers/system-helper' import { createSetupStatus } from './setup-status' import { SetupUI } from './setup-ui' /** * Download and set up llama.cpp * 1. Resolve the release version from versions.json * 2. Build from source on Linux + CUDA when required * 3. Otherwise download the matching prebuilt archive * 4. Keep the final binaries in their stable runtime directory */ const MOVE_FALLBACK_ERROR_CODES = new Set(['EXDEV', 'EPERM', 'EBUSY', 'EACCES']) const { cpuArchitecture: CPU_ARCH } = SystemHelper.getInformation() const LLAMA_SERVER_BINARY_NAME = SystemHelper.isWindows() ? 'llama-server.exe' : 'llama-server' const LLAMACPP_SOURCE_DOWNLOAD_MAX_ATTEMPTS = 2 const LLAMACPP_SOURCE_ARCHIVE_SETTLE_DELAY_MS = 500 const LLAMACPP_SOURCE_ARCHIVE_SETTLE_POLL_DELAY_MS = 250 const LLAMACPP_SOURCE_ARCHIVE_SETTLE_MAX_POLLS = 6 const LLAMACPP_RELEASE_BASE_URL = `https://github.com/ggml-org/llama.cpp/releases/download/${LLAMACPP_RELEASE_VERSION}` const LLAMACPP_SOURCE_URL = `https://github.com/ggml-org/llama.cpp/archive/refs/tags/${LLAMACPP_RELEASE_VERSION}.tar.gz` function readManifest() { const manifestEntries = [ { manifestPath: LLAMACPP_SOURCE_MANIFEST_PATH, runtimeBasePath: LLAMACPP_SOURCE_PATH }, { manifestPath: LLAMACPP_BUILD_MANIFEST_PATH, runtimeBasePath: LLAMACPP_BUILD_PATH }, // Keep compatibility with the previous root-level manifest layout. { manifestPath: LLAMACPP_ROOT_MANIFEST_PATH, runtimeBasePath: LLAMACPP_PATH } ] for (const manifestEntry of manifestEntries) { if (!fs.existsSync(manifestEntry.manifestPath)) { continue } try { const manifest = JSON.parse( fs.readFileSync(manifestEntry.manifestPath, 'utf8') ) const runtimeDirectoryPath = typeof manifest.runtimePath === 'string' && manifest.runtimePath.trim() ? path.join(manifestEntry.runtimeBasePath, manifest.runtimePath) : null return { manifest, runtimeDirectoryPath } } catch { return null } } return null } async function removePath(targetPath) { await fs.promises.rm(targetPath, { recursive: true, force: true }) } function wait(delayMs) { return new Promise((resolve) => { setTimeout(resolve, delayMs) }) } function isMoveFallbackError(error) { return ( error instanceof Error && 'code' in error && MOVE_FALLBACK_ERROR_CODES.has(error.code) ) } async function movePath(sourcePath, destinationPath) { await removePath(destinationPath) try { await fs.promises.rename(sourcePath, destinationPath) } catch (error) { if (!isMoveFallbackError(error)) { throw error } await fs.promises.cp(sourcePath, destinationPath, { recursive: true, force: true }) await removePath(sourcePath) } } function getBinaryPath(directoryPath) { return path.join(directoryPath, LLAMA_SERVER_BINARY_NAME) } async function isExistingInstallationHealthy(runtimeDirectoryPath) { if (!runtimeDirectoryPath) { return false } const binaryPath = getBinaryPath(runtimeDirectoryPath) if (!fs.existsSync(binaryPath)) { return false } try { await execa(binaryPath, ['--version']) return true } catch { return false } } async function findDirectoryContainingBinary(rootPath, binaryName) { const entries = await fs.promises.readdir(rootPath, { withFileTypes: true }) for (const entry of entries) { const entryPath = path.join(rootPath, entry.name) if (entry.isDirectory()) { const maybeBinDir = await findDirectoryContainingBinary( entryPath, binaryName ) if (maybeBinDir) { return maybeBinDir } } else if (entry.isFile() && entry.name === binaryName) { return path.dirname(entryPath) } } return null } async function cleanInstallDirectory() { await fs.promises.mkdir(LLAMACPP_PATH, { recursive: true }) const entries = await fs.promises.readdir(LLAMACPP_PATH, { withFileTypes: true }) await Promise.all( entries .filter((entry) => entry.name !== 'versions.json') .map((entry) => removePath(path.join(LLAMACPP_PATH, entry.name))) ) } async function writeManifest( manifestPath, runtimeBasePath, runtimeDirectoryPath, extraData = {} ) { await fs.promises.mkdir(path.dirname(manifestPath), { recursive: true }) await FileHelper.createManifestFile( manifestPath, 'llama.cpp', LLAMACPP_RELEASE_VERSION, { runtimePath: path.relative(runtimeBasePath, runtimeDirectoryPath), os: SystemHelper.getInformation().type, architecture: SystemHelper.getInformation().cpuArchitecture, ...extraData } ) } async function pruneSourceTree() { const buildDirectoryPath = path.join(LLAMACPP_SOURCE_PATH, 'build') const temporaryRetainedPath = await fs.promises.mkdtemp( path.join(LLAMACPP_PATH, 'llama-cpp-build-bin-') ) const retainedBuildBinPath = path.join(temporaryRetainedPath, 'bin') await movePath(LLAMACPP_SOURCE_BUILD_PATH, retainedBuildBinPath) await removePath(LLAMACPP_SOURCE_PATH) await fs.promises.mkdir(buildDirectoryPath, { recursive: true }) await movePath(retainedBuildBinPath, LLAMACPP_SOURCE_BUILD_PATH) await removePath(temporaryRetainedPath) } async function waitForArchiveToSettle(archivePath) { // Give the download layer a short margin, then wait until the archive size // stops changing before extracting it. await wait(LLAMACPP_SOURCE_ARCHIVE_SETTLE_DELAY_MS) let previousSize = -1 for (let poll = 0; poll < LLAMACPP_SOURCE_ARCHIVE_SETTLE_MAX_POLLS; poll += 1) { const currentSize = (await fs.promises.stat(archivePath)).size if (currentSize > 0 && currentSize === previousSize) { return } previousSize = currentSize await wait(LLAMACPP_SOURCE_ARCHIVE_SETTLE_POLL_DELAY_MS) } } async function downloadAndExtractSourceArchive(sourceArchivePath) { for ( let attempt = 1; attempt <= LLAMACPP_SOURCE_DOWNLOAD_MAX_ATTEMPTS; attempt += 1 ) { await Promise.all([ removePath(sourceArchivePath), removePath(LLAMACPP_SOURCE_PATH) ]) await FileHelper.downloadFile(LLAMACPP_SOURCE_URL, sourceArchivePath, { cliProgress: true, // Keep the source archive download conservative to avoid corrupted // segmented downloads before extraction. parallelStreams: 1, skipExisting: false }) await waitForArchiveToSettle(sourceArchivePath) try { await FileHelper.extractArchive(sourceArchivePath, LLAMACPP_SOURCE_PATH, { stripComponents: 1 }) return } catch (error) { if (attempt === LLAMACPP_SOURCE_DOWNLOAD_MAX_ATTEMPTS) { throw error } await wait(LLAMACPP_SOURCE_ARCHIVE_SETTLE_POLL_DELAY_MS) } } } function getLinuxVulkanAssetName() { return `llama-${LLAMACPP_RELEASE_VERSION}-bin-ubuntu-vulkan-x64.tar.gz` } function getLinuxCPUAssetName() { return `llama-${LLAMACPP_RELEASE_VERSION}-bin-ubuntu-x64.tar.gz` } function getWindowsCPUAssetName() { return `llama-${LLAMACPP_RELEASE_VERSION}-bin-win-cpu-x64.zip` } function getPrebuiltAssetName(graphicsComputeAPI, hasGPU) { if (SystemHelper.isMacOS()) { return CPU_ARCH === CPUArchitectures.ARM64 ? `llama-${LLAMACPP_RELEASE_VERSION}-bin-macos-arm64.tar.gz` : `llama-${LLAMACPP_RELEASE_VERSION}-bin-macos-x64.tar.gz` } if (SystemHelper.isWindows()) { if (hasGPU && graphicsComputeAPI === 'cuda') { return `llama-${LLAMACPP_RELEASE_VERSION}-bin-win-cuda-12.4-x64.zip` } if (hasGPU && graphicsComputeAPI === 'vulkan') { return `llama-${LLAMACPP_RELEASE_VERSION}-bin-win-vulkan-x64.zip` } return getWindowsCPUAssetName() } if (SystemHelper.isLinux() && CPU_ARCH === CPUArchitectures.X64) { return hasGPU && graphicsComputeAPI === 'vulkan' ? getLinuxVulkanAssetName() : getLinuxCPUAssetName() } throw new Error( `Unsupported llama.cpp prebuilt platform: ${SystemHelper.getInformation().type} ${CPU_ARCH}` ) } async function installPrebuilt(assetName, status, extraData = {}) { const archivePath = path.join(LLAMACPP_PATH, assetName) try { await cleanInstallDirectory() status.pause() await FileHelper.downloadFile( `${LLAMACPP_RELEASE_BASE_URL}/${assetName}`, archivePath, { cliProgress: true, parallelStreams: 3, skipExisting: false } ) status.text = 'Installing llama.cpp...' status.start() await FileHelper.extractArchive(archivePath, LLAMACPP_BUILD_PATH) // Use the directory that actually contains llama-server so we do not rely // on a fixed archive layout across upstream release assets. const binaryDirectoryPath = await findDirectoryContainingBinary( LLAMACPP_BUILD_PATH, LLAMA_SERVER_BINARY_NAME ) if (!binaryDirectoryPath) { throw new Error( `Cannot find ${LLAMA_SERVER_BINARY_NAME} in extracted llama.cpp archive` ) } await writeManifest( LLAMACPP_BUILD_MANIFEST_PATH, LLAMACPP_BUILD_PATH, binaryDirectoryPath, { installType: 'prebuilt', ...extraData } ) status.succeed(`llama.cpp ${LLAMACPP_RELEASE_VERSION} ready`) } finally { await removePath(archivePath) } } async function buildFromSource(status) { const sourceArchivePath = path.join( LLAMACPP_PATH, `llama.cpp-${LLAMACPP_RELEASE_VERSION}.tar.gz` ) try { await cleanInstallDirectory() status.pause() await downloadAndExtractSourceArchive(sourceArchivePath) status.text = 'Building llama.cpp from source...' status.start() // Always use Leon-managed CMake for the source build. await execa( CMAKE_BIN_PATH, [ '-B', 'build', '-G', 'Ninja', `-DCMAKE_MAKE_PROGRAM=${NINJA_BIN_PATH}`, '-DGGML_CUDA=ON', '-DLLAMA_BUILD_SERVER=ON', '-DLLAMA_BUILD_TESTS=OFF', '-DLLAMA_BUILD_EXAMPLES=OFF', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_CUDA_ARCHITECTURES=native' ], { cwd: LLAMACPP_SOURCE_PATH } ) await execa(CMAKE_BIN_PATH, ['--build', 'build', '--target', 'llama-server', '-j'], { cwd: LLAMACPP_SOURCE_PATH }) if ( !fs.existsSync(getBinaryPath(LLAMACPP_SOURCE_BUILD_PATH)) ) { throw new Error( `Cannot find ${LLAMA_SERVER_BINARY_NAME} after building llama.cpp` ) } // Retain only the runtime payload after a successful source build. await pruneSourceTree() await writeManifest( LLAMACPP_SOURCE_MANIFEST_PATH, LLAMACPP_SOURCE_PATH, LLAMACPP_SOURCE_BUILD_PATH, { installType: 'source' } ) status.succeed(`llama.cpp ${LLAMACPP_RELEASE_VERSION} ready`) } finally { await removePath(sourceArchivePath) } } export default async function setupLlamaCPP() { const status = createSetupStatus('Setting up llama.cpp...').start() const existingInstallation = readManifest() const manifest = existingInstallation?.manifest const runtimeDirectoryPath = existingInstallation?.runtimeDirectoryPath || null if ( manifest?.version === LLAMACPP_RELEASE_VERSION && (await isExistingInstallationHealthy(runtimeDirectoryPath)) ) { status.succeed(`llama.cpp: ${LLAMACPP_RELEASE_VERSION}`) return true } if ( manifest?.version === LLAMACPP_RELEASE_VERSION && runtimeDirectoryPath && fs.existsSync(getBinaryPath(runtimeDirectoryPath)) ) { status.pause() SetupUI.warning('The current llama.cpp installation looks corrupted. Reinstalling it.') } let hasGPU = false let graphicsComputeAPI = 'cpu' try { const { getLlama, LlamaLogLevel } = await Function( 'return import("node-llama-cpp")' )() const llama = await getLlama({ logLevel: LlamaLogLevel.disabled }) hasGPU = await SystemHelper.hasGPU(llama) graphicsComputeAPI = await SystemHelper.getGraphicsComputeAPI(llama) } catch (error) { status.pause() SetupUI.warning(`Failed to inspect GPU support for llama.cpp setup: ${error}`) } if (SystemHelper.isLinux() && CPU_ARCH === CPUArchitectures.ARM64) { // Linux ARM64 only supports the local setup when a CUDA build is possible. if (!(hasGPU && graphicsComputeAPI === 'cuda')) { status.succeed( 'Linux ARM64 local LLM support requires a CUDA GPU. Skipping llama.cpp setup.' ) return false } try { await buildFromSource(status) return true } catch { if (status.isSpinning) { status.fail('Failed to build llama.cpp from source') } return false } } if (SystemHelper.isLinux() && hasGPU && graphicsComputeAPI === 'cuda') { try { await buildFromSource(status) return true } catch { SetupUI.warning( 'I could not build llama.cpp from source, so I will use the Vulkan binaries instead.' ) await installPrebuilt(getLinuxVulkanAssetName(), status, { fallbackFromSourceBuild: true }) return true } } await installPrebuilt( getPrebuiltAssetName(graphicsComputeAPI, hasGPU), status ) return true }