feat: add SettingsPage with HuggingFace model search and fix flexbox layout
- Add SettingsPage component with macOS/Linear-style settings UI (sp-* classes) - Fix HuggingFace results list squeeze: sp-card-row--full now has align-items: stretch overriding the inherited flex-start, ensuring children fill 100% horizontal width - Fix sp-hf-file row layout: width 100%, grid columns for name/quant/size/download - Fix sp-hf-name truncation: explicit overflow/ellipsis/white-space in CSS - Add white-space: nowrap to sp-hf-size and sp-hf-quant-badge to protect secondary columns - Add runtime-deps crate, Tailwind/PostCSS config, runtime-defaults, MessageBubble test - Update Cargo workspace, inference-server, tauri runtime types and process handling
This commit is contained in:
Evandro Fonseca Junior committed
1 parent
9da25c3688
commit
01b629cb4a
41 files changed
+6864
-822
No files matched your search
@@ -11,29 +11,34 @@ pub enum RuntimeProfile {
|
||||
|
||||
impl RuntimeProfile {
|
||||
/// Returns sensible defaults for each profile.
|
||||
///
|
||||
/// `n_ctx` sets the KV cache / context window size. Larger values allow longer
|
||||
/// conversations but consume more VRAM. Override at runtime with `LOCALLLM_CTX`.
|
||||
/// `n_batch` controls the prefill chunk size — kept independent of `n_ctx` to
|
||||
/// avoid pre-allocating the full window as a single batch buffer (OOM risk).
|
||||
pub fn defaults(&self) -> ProfileDefaults {
|
||||
match self {
|
||||
Self::Turbo4B => ProfileDefaults {
|
||||
n_gpu_layers: 999,
|
||||
n_ctx: 4096,
|
||||
n_batch: 512,
|
||||
n_ubatch: 128,
|
||||
n_ctx: 131_072, // 128 K — covers very long conversations; needs ~4 GB KV VRAM at FP16
|
||||
n_batch: 2048, // prefill chunk size; independent of n_ctx
|
||||
n_ubatch: 512,
|
||||
flash_attn: true,
|
||||
n_threads: 8,
|
||||
},
|
||||
Self::Balanced9B => ProfileDefaults {
|
||||
n_gpu_layers: 999,
|
||||
n_ctx: 2048,
|
||||
n_batch: 256,
|
||||
n_ubatch: 64,
|
||||
n_ctx: 65_536, // 64 K
|
||||
n_batch: 2048,
|
||||
n_ubatch: 256,
|
||||
flash_attn: true,
|
||||
n_threads: 8,
|
||||
},
|
||||
Self::CpuFallback => ProfileDefaults {
|
||||
n_gpu_layers: 0,
|
||||
n_ctx: 2048,
|
||||
n_batch: 128,
|
||||
n_ubatch: 32,
|
||||
n_ctx: 16_384, // 16 K — CPU KV cache is RAM-backed; keep conservative
|
||||
n_batch: 512,
|
||||
n_ubatch: 128,
|
||||
flash_attn: false,
|
||||
n_threads: num_cpus(),
|
||||
},
|
||||
|
||||
@@ -12,6 +12,8 @@ path = "src/main.rs"
|
||||
inference-core = { path = "../inference-core" }
|
||||
api-contracts = { path = "../api-contracts" }
|
||||
metrics = { path = "../metrics" }
|
||||
runtime-deps = { path = "../runtime-deps" }
|
||||
dirs = { workspace = true }
|
||||
tokio.workspace = true
|
||||
axum.workspace = true
|
||||
hyper.workspace = true
|
||||
|
||||
@@ -1,38 +1,4 @@
|
||||
fn main() {
|
||||
let out_dir = std::env::var("OUT_DIR").unwrap();
|
||||
let target_dir = std::path::Path::new(&out_dir)
|
||||
.ancestors()
|
||||
.nth(3) // Volta 3 níveis de `target/debug/build/inference-server-XYZ/out` para `target/debug/` ou `target/release/`
|
||||
.unwrap();
|
||||
|
||||
let vendor_dir = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("vendor")
|
||||
.join("llama-bin");
|
||||
|
||||
if vendor_dir.exists() {
|
||||
println!(
|
||||
"cargo:warning=Copying DLLs from {:?} to {:?}",
|
||||
vendor_dir, target_dir
|
||||
);
|
||||
|
||||
// Vamos copiar todos os arquivos .dll
|
||||
if let Ok(entries) = std::fs::read_dir(&vendor_dir) {
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("dll") {
|
||||
let file_name = path.file_name().unwrap();
|
||||
let dest = target_dir.join(file_name);
|
||||
let _ = std::fs::copy(&path, &dest);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuildar apenas se houver mudanças nessa build file ou nos bins da vendor
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
println!("cargo:rerun-if-changed=../../vendor/llama-bin");
|
||||
println!("cargo:rerun-if-changed=../../vendor/llama-runtime/manifest.json");
|
||||
}
|
||||
@@ -3,8 +3,12 @@ use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use runtime_deps::{
|
||||
apply_runtime_to_current_process, ensure_llama_runtime, recommended_variant_override_hint,
|
||||
EnsureRuntimeRequest,
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing_subscriber::{fmt, EnvFilter};
|
||||
use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
|
||||
|
||||
use inference_core::{EngineConfig, InferenceEngine, RuntimeProfile};
|
||||
use metrics::counters::PerfCounters;
|
||||
@@ -17,12 +21,25 @@ use routes::{build_router, AppState};
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// ── Tracing ──────────────────────────────────────────────────────────
|
||||
fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.with_target(false)
|
||||
.init();
|
||||
// Set LOCALLLM_LOG_FORMAT=json for machine-readable output (e.g. log aggregators).
|
||||
// Set RUST_LOG=debug for verbose output.
|
||||
let env_filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
|
||||
match std::env::var("LOCALLLM_LOG_FORMAT").as_deref() {
|
||||
Ok("json") => {
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(fmt::layer().json().with_current_span(true).with_span_list(true))
|
||||
.init();
|
||||
}
|
||||
_ => {
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(fmt::layer().with_target(true))
|
||||
.init();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Engine config (reads env vars; sane defaults for dev) ────────────
|
||||
let model_path = std::env::var("LOCALLLM_MODEL")
|
||||
@@ -62,6 +79,45 @@ async fn main() -> Result<()> {
|
||||
let server_config = Arc::new(ServerConfig::from_env(model_name.clone()));
|
||||
|
||||
// ── Load engine ──────────────────────────────────────────────────────
|
||||
tracing::info!(
|
||||
model = %model_path.display(),
|
||||
profile = ?profile,
|
||||
n_gpu_layers = config.n_gpu_layers,
|
||||
n_ctx = config.n_ctx,
|
||||
n_batch = config.n_batch,
|
||||
n_ubatch = config.n_ubatch,
|
||||
n_threads = config.n_threads,
|
||||
n_threads_batch = config.n_threads_batch,
|
||||
flash_attn = config.flash_attn,
|
||||
use_mmap = config.use_mmap,
|
||||
"engine_config"
|
||||
);
|
||||
let binary_path = std::env::current_exe()?;
|
||||
let runtime_cache_root = dirs::data_local_dir()
|
||||
.unwrap_or_else(|| {
|
||||
binary_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.to_path_buf()
|
||||
})
|
||||
.join("ForgeLM");
|
||||
let runtime = ensure_llama_runtime(&EnsureRuntimeRequest {
|
||||
cache_root: &runtime_cache_root,
|
||||
gpu_layers: config.n_gpu_layers,
|
||||
variant_override: None,
|
||||
binary_path: Some(&binary_path),
|
||||
})
|
||||
.map_err(|error| anyhow::anyhow!(
|
||||
"failed to prepare llama runtime dependencies: {error}. {}",
|
||||
recommended_variant_override_hint()
|
||||
))?;
|
||||
apply_runtime_to_current_process(&runtime);
|
||||
tracing::info!(
|
||||
dll = %runtime.dll_path.display(),
|
||||
variant = runtime.variant_id,
|
||||
source = ?runtime.source,
|
||||
"llama_runtime_ready"
|
||||
);
|
||||
let mut engine = InferenceEngine::load(config)?;
|
||||
engine.warmup()?;
|
||||
|
||||
@@ -81,6 +137,14 @@ async fn main() -> Result<()> {
|
||||
let router = build_router(state);
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
|
||||
tracing::info!(
|
||||
bind_addr = %addr,
|
||||
model = %model_name,
|
||||
api_key_set = std::env::var("LOCALLLM_API_KEY").is_ok(),
|
||||
log_format = std::env::var("LOCALLLM_LOG_FORMAT").as_deref().unwrap_or("pretty"),
|
||||
"server_ready"
|
||||
);
|
||||
|
||||
// ── Startup banner ───────────────────────────────────────────────────
|
||||
let display_addr = if addr.ip().is_unspecified() {
|
||||
format!("http://127.0.0.1:{}", addr.port())
|
||||
@@ -125,8 +189,20 @@ fn env_u32(key: &str, default: u32) -> u32 {
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn resolve_batch_size(n_ctx: u32, default_n_batch: u32, explicit_batch: Option<u32>) -> u32 {
|
||||
explicit_batch.unwrap_or_else(|| n_ctx.max(default_n_batch).max(1))
|
||||
/// Choose the prefill batch size.
|
||||
///
|
||||
/// The batch size controls how many tokens are processed in a single `llama_decode`
|
||||
/// call during prefill. It must not be confused with the context window (`n_ctx`):
|
||||
/// a large `n_ctx` does NOT require an equally large `n_batch` — `decode_prefill`
|
||||
/// already chunks the prompt into `n_batch`-sized slices.
|
||||
///
|
||||
/// Setting `n_batch = n_ctx` for a 128 K context would pre-allocate ~128 K token
|
||||
/// slots in the batch buffer, consuming gigabytes of VRAM at startup before any
|
||||
/// request arrives. Instead we cap auto-selection at `MAX_AUTO_BATCH` and let
|
||||
/// callers pass `LOCALLLM_BATCH` when they need a larger explicit override.
|
||||
fn resolve_batch_size(_n_ctx: u32, default_n_batch: u32, explicit_batch: Option<u32>) -> u32 {
|
||||
const MAX_AUTO_BATCH: u32 = 4096;
|
||||
explicit_batch.unwrap_or_else(|| default_n_batch.min(MAX_AUTO_BATCH).max(1))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -134,8 +210,11 @@ mod tests {
|
||||
use super::resolve_batch_size;
|
||||
|
||||
#[test]
|
||||
fn defaults_batch_to_context_size_when_ctx_is_larger() {
|
||||
assert_eq!(resolve_batch_size(16_384, 512, None), 16_384);
|
||||
fn caps_auto_batch_at_max_regardless_of_ctx_size() {
|
||||
// Large n_ctx must NOT bloat n_batch — that would OOM the batch buffer.
|
||||
assert_eq!(resolve_batch_size(131_072, 512, None), 512);
|
||||
assert_eq!(resolve_batch_size(131_072, 4096, None), 4096);
|
||||
assert_eq!(resolve_batch_size(131_072, 8192, None), 4096); // capped at MAX_AUTO_BATCH
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -397,9 +397,17 @@ async fn stream_response(
|
||||
}
|
||||
};
|
||||
|
||||
Sse::new(stream)
|
||||
let response = Sse::new(stream)
|
||||
.keep_alive(KeepAlive::default())
|
||||
.into_response()
|
||||
.into_response();
|
||||
|
||||
axum::response::Response::builder()
|
||||
.status(response.status())
|
||||
.header("content-type", "text/event-stream; charset=utf-8")
|
||||
.header("cache-control", "no-cache")
|
||||
.header("connection", "keep-alive")
|
||||
.body(response.into_body())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn blocking_response(
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "runtime-deps"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
ureq = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
dirs = { workspace = true }
|
||||
@@ -0,0 +1,614 @@
|
||||
use std::ffi::OsString;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, BufReader, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const MANIFEST_JSON: &str = include_str!("../../../vendor/llama-runtime/manifest.json");
|
||||
const LOCKFILE_NAME: &str = ".install.lock";
|
||||
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const LOCK_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnsureRuntimeRequest<'a> {
|
||||
pub cache_root: &'a Path,
|
||||
pub gpu_layers: i32,
|
||||
pub variant_override: Option<&'a str>,
|
||||
pub binary_path: Option<&'a Path>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResolvedRuntime {
|
||||
pub dll_path: PathBuf,
|
||||
pub library_dir: PathBuf,
|
||||
pub variant_id: String,
|
||||
pub source: RuntimeSource,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RuntimeSource {
|
||||
EnvOverride,
|
||||
AdjacentBinary,
|
||||
LegacyVendor,
|
||||
ManagedCache,
|
||||
}
|
||||
|
||||
pub fn ensure_llama_runtime(request: &EnsureRuntimeRequest<'_>) -> Result<ResolvedRuntime> {
|
||||
if let Some(runtime) = resolve_from_env_override()? {
|
||||
return Ok(runtime);
|
||||
}
|
||||
|
||||
let manifest = load_manifest()?;
|
||||
let variant = select_variant(&manifest, request.gpu_layers, request.variant_override)?;
|
||||
|
||||
if let Some(runtime) = resolve_from_adjacent_binary(request.binary_path, variant)? {
|
||||
return Ok(runtime);
|
||||
}
|
||||
|
||||
if let Some(runtime) = resolve_from_legacy_vendor(request.binary_path, variant)? {
|
||||
return Ok(runtime);
|
||||
}
|
||||
let install_dir = request
|
||||
.cache_root
|
||||
.join("llama-runtime")
|
||||
.join(&manifest.release)
|
||||
.join(&variant.id);
|
||||
let dll_path = install_dir.join(&variant.dll);
|
||||
|
||||
if installation_is_valid(&install_dir, variant) {
|
||||
return Ok(ResolvedRuntime {
|
||||
dll_path,
|
||||
library_dir: install_dir,
|
||||
variant_id: variant.id.clone(),
|
||||
source: RuntimeSource::ManagedCache,
|
||||
});
|
||||
}
|
||||
|
||||
fs::create_dir_all(install_dir.parent().unwrap_or(request.cache_root)).with_context(|| {
|
||||
format!(
|
||||
"failed to create runtime cache root at {}",
|
||||
request.cache_root.display()
|
||||
)
|
||||
})?;
|
||||
let _lock = InstallationLock::acquire(&install_dir)?;
|
||||
|
||||
if installation_is_valid(&install_dir, variant) {
|
||||
return Ok(ResolvedRuntime {
|
||||
dll_path,
|
||||
library_dir: install_dir,
|
||||
variant_id: variant.id.clone(),
|
||||
source: RuntimeSource::ManagedCache,
|
||||
});
|
||||
}
|
||||
|
||||
install_variant(&install_dir, variant)?;
|
||||
|
||||
Ok(ResolvedRuntime {
|
||||
dll_path,
|
||||
library_dir: install_dir,
|
||||
variant_id: variant.id.clone(),
|
||||
source: RuntimeSource::ManagedCache,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_runtime_to_command(command: &mut Command, runtime: &ResolvedRuntime) {
|
||||
command.env("LLAMA_DLL_PATH", &runtime.dll_path);
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let path = prepend_path_entry(&runtime.library_dir, std::env::var_os("PATH"));
|
||||
command.env("PATH", path);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_runtime_to_current_process(runtime: &ResolvedRuntime) {
|
||||
std::env::set_var("LLAMA_DLL_PATH", &runtime.dll_path);
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let path = prepend_path_entry(&runtime.library_dir, std::env::var_os("PATH"));
|
||||
std::env::set_var("PATH", path);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recommended_variant_override_hint() -> &'static str {
|
||||
"Set FORGELM_LLAMA_RUNTIME_VARIANT to a manifest variant id like windows-x64-cpu or windows-x64-cuda-12.4."
|
||||
}
|
||||
|
||||
fn resolve_from_env_override() -> Result<Option<ResolvedRuntime>> {
|
||||
let Ok(raw_path) = std::env::var("LLAMA_DLL_PATH") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let dll_path = PathBuf::from(raw_path);
|
||||
if !dll_path.exists() {
|
||||
bail!(
|
||||
"LLAMA_DLL_PATH points to {}, but that file does not exist",
|
||||
dll_path.display()
|
||||
);
|
||||
}
|
||||
let library_dir = dll_path
|
||||
.parent()
|
||||
.map(Path::to_path_buf)
|
||||
.ok_or_else(|| anyhow!("LLAMA_DLL_PATH must point to a file, not a root path"))?;
|
||||
Ok(Some(ResolvedRuntime {
|
||||
dll_path,
|
||||
library_dir,
|
||||
variant_id: "env-override".to_string(),
|
||||
source: RuntimeSource::EnvOverride,
|
||||
}))
|
||||
}
|
||||
|
||||
fn resolve_from_adjacent_binary(
|
||||
binary_path: Option<&Path>,
|
||||
variant: &ManifestVariant,
|
||||
) -> Result<Option<ResolvedRuntime>> {
|
||||
let Some(binary_path) = binary_path else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(binary_dir) = binary_path.parent() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let dll_path = binary_dir.join(default_dll_name());
|
||||
if dll_path.exists() && verify_required_files(binary_dir, variant).is_ok() {
|
||||
return Ok(Some(ResolvedRuntime {
|
||||
dll_path,
|
||||
library_dir: binary_dir.to_path_buf(),
|
||||
variant_id: "adjacent-binary".to_string(),
|
||||
source: RuntimeSource::AdjacentBinary,
|
||||
}));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn resolve_from_legacy_vendor(
|
||||
binary_path: Option<&Path>,
|
||||
variant: &ManifestVariant,
|
||||
) -> Result<Option<ResolvedRuntime>> {
|
||||
let Some(binary_path) = binary_path else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut dir = binary_path.parent();
|
||||
while let Some(current) = dir {
|
||||
let candidate = current
|
||||
.join("vendor")
|
||||
.join("llama-bin")
|
||||
.join(default_dll_name());
|
||||
let Some(library_dir) = candidate.parent() else {
|
||||
dir = current.parent();
|
||||
continue;
|
||||
};
|
||||
if candidate.exists() && verify_required_files(library_dir, variant).is_ok() {
|
||||
return Ok(Some(ResolvedRuntime {
|
||||
dll_path: candidate.clone(),
|
||||
library_dir: library_dir.to_path_buf(),
|
||||
variant_id: "legacy-vendor".to_string(),
|
||||
source: RuntimeSource::LegacyVendor,
|
||||
}));
|
||||
}
|
||||
dir = current.parent();
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn load_manifest() -> Result<Manifest> {
|
||||
serde_json::from_str(MANIFEST_JSON).context("failed to parse embedded llama runtime manifest")
|
||||
}
|
||||
|
||||
fn select_variant<'a>(
|
||||
manifest: &'a Manifest,
|
||||
gpu_layers: i32,
|
||||
variant_override: Option<&str>,
|
||||
) -> Result<&'a ManifestVariant> {
|
||||
let env_override = std::env::var("FORGELM_LLAMA_RUNTIME_VARIANT").ok();
|
||||
if let Some(override_id) = variant_override.or(env_override.as_deref()) {
|
||||
return manifest
|
||||
.variants
|
||||
.iter()
|
||||
.find(|variant| variant.id == override_id)
|
||||
.ok_or_else(|| anyhow!("unknown runtime variant '{override_id}'"));
|
||||
}
|
||||
|
||||
let target = current_target();
|
||||
let want_gpu = gpu_layers > 0;
|
||||
manifest
|
||||
.variants
|
||||
.iter()
|
||||
.find(|variant| variant.target == target && variant.gpu == want_gpu)
|
||||
.or_else(|| {
|
||||
manifest
|
||||
.variants
|
||||
.iter()
|
||||
.find(|variant| variant.target == target && !variant.gpu)
|
||||
})
|
||||
.ok_or_else(|| anyhow!("no runtime bundle configured for target {target}"))
|
||||
}
|
||||
|
||||
fn install_variant(install_dir: &Path, variant: &ManifestVariant) -> Result<()> {
|
||||
if let Some(parent) = install_dir.parent() {
|
||||
fs::create_dir_all(parent).with_context(|| {
|
||||
format!(
|
||||
"failed to create runtime install parent at {}",
|
||||
parent.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
let staging_dir = install_dir.with_extension("staging");
|
||||
if staging_dir.exists() {
|
||||
fs::remove_dir_all(&staging_dir).with_context(|| {
|
||||
format!(
|
||||
"failed to clean previous staging dir {}",
|
||||
staging_dir.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
fs::create_dir_all(&staging_dir)
|
||||
.with_context(|| format!("failed to create staging dir {}", staging_dir.display()))?;
|
||||
|
||||
for archive in &variant.archives {
|
||||
let archive_path = download_archive_to_file(archive, &staging_dir)?;
|
||||
extract_zip_file(&archive_path, &staging_dir)?;
|
||||
}
|
||||
|
||||
verify_required_files(&staging_dir, variant)?;
|
||||
|
||||
if install_dir.exists() {
|
||||
fs::remove_dir_all(install_dir).with_context(|| {
|
||||
format!(
|
||||
"failed to replace old runtime dir {}",
|
||||
install_dir.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
fs::rename(&staging_dir, install_dir).or_else(|rename_error| {
|
||||
copy_dir_recursive(&staging_dir, install_dir)
|
||||
.and_then(|_| {
|
||||
fs::remove_dir_all(&staging_dir)
|
||||
.with_context(|| format!("failed to remove staging dir {}", staging_dir.display()))
|
||||
})
|
||||
.map_err(|copy_error| {
|
||||
anyhow!(
|
||||
"failed to finalize runtime install via rename ({rename_error}) and copy fallback ({copy_error})"
|
||||
)
|
||||
})
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn download_archive_to_file(archive: &ManifestArchive, staging_dir: &Path) -> Result<PathBuf> {
|
||||
let downloads_dir = staging_dir.join(".downloads");
|
||||
fs::create_dir_all(&downloads_dir)
|
||||
.with_context(|| format!("failed to create downloads dir {}", downloads_dir.display()))?;
|
||||
let archive_name = archive
|
||||
.url
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.filter(|name| !name.is_empty())
|
||||
.ok_or_else(|| anyhow!("archive url has no file name: {}", archive.url))?;
|
||||
let archive_path = downloads_dir.join(archive_name);
|
||||
|
||||
let response = ureq::get(&archive.url)
|
||||
.timeout(DOWNLOAD_TIMEOUT)
|
||||
.call()
|
||||
.with_context(|| format!("failed to download {}", archive.url))?;
|
||||
let mut reader = response.into_reader();
|
||||
let mut file = File::create(&archive_path)
|
||||
.with_context(|| format!("failed to create archive file {}", archive_path.display()))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut total_bytes = 0u64;
|
||||
let mut buffer = [0u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let read = reader
|
||||
.read(&mut buffer)
|
||||
.with_context(|| format!("failed to stream downloaded bytes from {}", archive.url))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
total_bytes += read as u64;
|
||||
if total_bytes > archive.size {
|
||||
bail!(
|
||||
"downloaded archive {} exceeded declared size {} bytes",
|
||||
archive.url,
|
||||
archive.size
|
||||
);
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
file.write_all(&buffer[..read]).with_context(|| {
|
||||
format!(
|
||||
"failed to write downloaded archive to {}",
|
||||
archive_path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
if total_bytes != archive.size {
|
||||
bail!(
|
||||
"downloaded archive {} size mismatch: expected {} bytes, got {}",
|
||||
archive.url,
|
||||
archive.size,
|
||||
total_bytes
|
||||
);
|
||||
}
|
||||
|
||||
let expected = &archive.sha256;
|
||||
let actual = format!("{:x}", hasher.finalize());
|
||||
if actual != *expected {
|
||||
bail!(
|
||||
"downloaded archive {} failed sha256 verification: expected {}, got {}",
|
||||
archive.url,
|
||||
expected,
|
||||
actual
|
||||
);
|
||||
}
|
||||
Ok(archive_path)
|
||||
}
|
||||
|
||||
fn extract_zip_file(archive_path: &Path, dest_dir: &Path) -> Result<()> {
|
||||
let file = File::open(archive_path)
|
||||
.with_context(|| format!("failed to open archive {}", archive_path.display()))?;
|
||||
let mut archive =
|
||||
zip::ZipArchive::new(BufReader::new(file)).context("failed to open zip archive")?;
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive
|
||||
.by_index(index)
|
||||
.context("failed to read zip entry")?;
|
||||
let Some(relative_path) = entry.enclosed_name().map(|path| path.to_path_buf()) else {
|
||||
continue;
|
||||
};
|
||||
let output_path = dest_dir.join(relative_path);
|
||||
if entry.is_dir() {
|
||||
fs::create_dir_all(&output_path).with_context(|| {
|
||||
format!("failed to create extracted dir {}", output_path.display())
|
||||
})?;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(parent) = output_path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("failed to create parent dir {}", parent.display()))?;
|
||||
}
|
||||
|
||||
let mut output = File::create(&output_path).with_context(|| {
|
||||
format!("failed to create extracted file {}", output_path.display())
|
||||
})?;
|
||||
io::copy(&mut entry, &mut output)
|
||||
.with_context(|| format!("failed to extract {}", output_path.display()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn verify_required_files(install_dir: &Path, variant: &ManifestVariant) -> Result<()> {
|
||||
for required_file in &variant.required_files {
|
||||
let candidate = install_dir.join(required_file);
|
||||
if !candidate.exists() {
|
||||
bail!(
|
||||
"runtime bundle {} is missing required file {}",
|
||||
variant.id,
|
||||
candidate.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn installation_is_valid(install_dir: &Path, variant: &ManifestVariant) -> bool {
|
||||
install_dir.exists() && verify_required_files(install_dir, variant).is_ok()
|
||||
}
|
||||
|
||||
fn default_dll_name() -> &'static str {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
"llama.dll"
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
"libllama.so"
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
"libllama.dylib"
|
||||
}
|
||||
}
|
||||
|
||||
fn current_target() -> &'static str {
|
||||
#[cfg(all(target_os = "windows", target_arch = "x86_64"))]
|
||||
{
|
||||
"windows-x86_64"
|
||||
}
|
||||
#[cfg(all(target_os = "windows", target_arch = "aarch64"))]
|
||||
{
|
||||
"windows-aarch64"
|
||||
}
|
||||
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
|
||||
{
|
||||
"linux-x86_64"
|
||||
}
|
||||
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
|
||||
{
|
||||
"macos-x86_64"
|
||||
}
|
||||
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
|
||||
{
|
||||
"macos-aarch64"
|
||||
}
|
||||
}
|
||||
|
||||
fn prepend_path_entry(entry: &Path, existing: Option<OsString>) -> OsString {
|
||||
let mut paths = vec![entry.to_path_buf()];
|
||||
if let Some(existing) = existing {
|
||||
paths.extend(std::env::split_paths(&existing));
|
||||
}
|
||||
std::env::join_paths(paths).unwrap_or_else(|_| OsString::from(entry.as_os_str()))
|
||||
}
|
||||
|
||||
fn copy_dir_recursive(source: &Path, destination: &Path) -> Result<()> {
|
||||
fs::create_dir_all(destination)
|
||||
.with_context(|| format!("failed to create {}", destination.display()))?;
|
||||
for entry in
|
||||
fs::read_dir(source).with_context(|| format!("failed to read {}", source.display()))?
|
||||
{
|
||||
let entry = entry?;
|
||||
let source_path = entry.path();
|
||||
let destination_path = destination.join(entry.file_name());
|
||||
if source_path.is_dir() {
|
||||
copy_dir_recursive(&source_path, &destination_path)?;
|
||||
} else {
|
||||
fs::copy(&source_path, &destination_path).with_context(|| {
|
||||
format!(
|
||||
"failed to copy {} to {}",
|
||||
source_path.display(),
|
||||
destination_path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct InstallationLock {
|
||||
path: PathBuf,
|
||||
_file: File,
|
||||
}
|
||||
|
||||
impl InstallationLock {
|
||||
fn acquire(install_dir: &Path) -> Result<Self> {
|
||||
fs::create_dir_all(install_dir.parent().unwrap_or(install_dir)).with_context(|| {
|
||||
format!("failed to create lock parent for {}", install_dir.display())
|
||||
})?;
|
||||
|
||||
let lock_path = install_dir.join(LOCKFILE_NAME);
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
match OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(&lock_path)
|
||||
{
|
||||
Ok(file) => {
|
||||
return Ok(Self {
|
||||
path: lock_path,
|
||||
_file: file,
|
||||
})
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
|
||||
if lock_is_stale(&lock_path)? {
|
||||
let _ = fs::remove_file(&lock_path);
|
||||
continue;
|
||||
}
|
||||
if started.elapsed() >= LOCK_TIMEOUT {
|
||||
bail!(
|
||||
"timed out waiting for runtime install lock at {}",
|
||||
lock_path.display()
|
||||
);
|
||||
}
|
||||
thread::sleep(LOCK_POLL_INTERVAL);
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(error).with_context(|| {
|
||||
format!(
|
||||
"failed to acquire runtime install lock at {}",
|
||||
lock_path.display()
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InstallationLock {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_is_stale(lock_path: &Path) -> Result<bool> {
|
||||
let metadata = match fs::metadata(lock_path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => {
|
||||
return Err(error).with_context(|| {
|
||||
format!("failed to read lock metadata for {}", lock_path.display())
|
||||
})
|
||||
}
|
||||
};
|
||||
let modified = metadata
|
||||
.modified()
|
||||
.with_context(|| format!("failed to read lock timestamp for {}", lock_path.display()))?;
|
||||
Ok(modified.elapsed().unwrap_or_default() > LOCK_TIMEOUT)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Manifest {
|
||||
release: String,
|
||||
variants: Vec<ManifestVariant>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManifestVariant {
|
||||
id: String,
|
||||
target: String,
|
||||
gpu: bool,
|
||||
dll: String,
|
||||
archives: Vec<ManifestArchive>,
|
||||
required_files: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManifestArchive {
|
||||
url: String,
|
||||
sha256: String,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{current_target, load_manifest, prepend_path_entry, select_variant, Manifest};
|
||||
use std::ffi::OsString;
|
||||
use std::path::Path;
|
||||
|
||||
fn manifest() -> Manifest {
|
||||
load_manifest().expect("manifest")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_cpu_variant_for_local_cpu_mode() {
|
||||
let manifest = manifest();
|
||||
let variant = select_variant(&manifest, 0, None).expect("variant");
|
||||
assert_eq!(variant.target, current_target());
|
||||
assert!(!variant.gpu);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_gpu_variant_when_gpu_layers_are_requested() {
|
||||
let manifest = manifest();
|
||||
let variant = select_variant(&manifest, 32, None).expect("variant");
|
||||
assert_eq!(variant.target, current_target());
|
||||
assert!(variant.gpu);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_explicit_variant_override() {
|
||||
let manifest = manifest();
|
||||
let variant = select_variant(&manifest, 0, Some("windows-x64-cpu")).expect("variant");
|
||||
assert_eq!(variant.id, "windows-x64-cpu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepends_runtime_dir_to_existing_path() {
|
||||
let joined = prepend_path_entry(
|
||||
Path::new("C:/runtime"),
|
||||
Some(OsString::from("C:/windows;C:/tools")),
|
||||
);
|
||||
let rendered = joined.to_string_lossy();
|
||||
assert!(rendered.starts_with("C:/runtime"));
|
||||
assert!(rendered.contains("C:/windows"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user