Evandro Fonseca Junior
01b629cb4a
- Add SettingsPage component with macOS/Linear-style settings UI (sp-* classes) - Fix HuggingFace results list squeeze: sp-card-row--full now has align-items: stretch overriding the inherited flex-start, ensuring children fill 100% horizontal width - Fix sp-hf-file row layout: width 100%, grid columns for name/quant/size/download - Fix sp-hf-name truncation: explicit overflow/ellipsis/white-space in CSS - Add white-space: nowrap to sp-hf-size and sp-hf-quant-badge to protect secondary columns - Add runtime-deps crate, Tailwind/PostCSS config, runtime-defaults, MessageBubble test - Update Cargo workspace, inference-server, tauri runtime types and process handling
65 lines
2.2 KiB
Rust
65 lines
2.2 KiB
Rust
/// Predefined runtime profiles matching the plan targets.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
pub enum RuntimeProfile {
|
||
/// Qwen3.5-4B – maximum TPS, text-only, flash attention, moderate context.
|
||
Turbo4B,
|
||
/// Qwen3.5-9B – balanced quality/speed, tuned KV cache.
|
||
Balanced9B,
|
||
/// CPU-only fallback for machines without a GPU.
|
||
CpuFallback,
|
||
}
|
||
|
||
impl RuntimeProfile {
|
||
/// Returns sensible defaults for each profile.
|
||
///
|
||
/// `n_ctx` sets the KV cache / context window size. Larger values allow longer
|
||
/// conversations but consume more VRAM. Override at runtime with `LOCALLLM_CTX`.
|
||
/// `n_batch` controls the prefill chunk size — kept independent of `n_ctx` to
|
||
/// avoid pre-allocating the full window as a single batch buffer (OOM risk).
|
||
pub fn defaults(&self) -> ProfileDefaults {
|
||
match self {
|
||
Self::Turbo4B => ProfileDefaults {
|
||
n_gpu_layers: 999,
|
||
n_ctx: 131_072, // 128 K — covers very long conversations; needs ~4 GB KV VRAM at FP16
|
||
n_batch: 2048, // prefill chunk size; independent of n_ctx
|
||
n_ubatch: 512,
|
||
flash_attn: true,
|
||
n_threads: 8,
|
||
},
|
||
Self::Balanced9B => ProfileDefaults {
|
||
n_gpu_layers: 999,
|
||
n_ctx: 65_536, // 64 K
|
||
n_batch: 2048,
|
||
n_ubatch: 256,
|
||
flash_attn: true,
|
||
n_threads: 8,
|
||
},
|
||
Self::CpuFallback => ProfileDefaults {
|
||
n_gpu_layers: 0,
|
||
n_ctx: 16_384, // 16 K — CPU KV cache is RAM-backed; keep conservative
|
||
n_batch: 512,
|
||
n_ubatch: 128,
|
||
flash_attn: false,
|
||
n_threads: num_cpus(),
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
fn num_cpus() -> i32 {
|
||
// Simple fallback; replace with `num_cpus` crate if needed.
|
||
std::thread::available_parallelism()
|
||
.map(|n| n.get() as i32)
|
||
.unwrap_or(4)
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct ProfileDefaults {
|
||
pub n_gpu_layers: i32,
|
||
pub n_ctx: u32,
|
||
pub n_batch: u32,
|
||
pub n_ubatch: u32,
|
||
pub flash_attn: bool,
|
||
pub n_threads: i32,
|
||
}
|