fix(quick-1): bail on NULL KV cache and improve prefill error diagnostics
- engine.rs: new_session() now returns Err when llama_get_memory returns NULL instead of silently continuing, preventing decode corruption on 2nd request - session.rs: decode_prefill error includes token count for easier diagnosis
This commit is contained in:
1 parent
54a58ee9c7
commit
d9198795bf
2 files changed
+45
-27
No files matched your search
@@ -263,10 +263,6 @@ impl InferenceEngine {
|
||||
info!("Running engine warmup (dummy decode)…");
|
||||
|
||||
unsafe {
|
||||
// Mark context as warmup so llama.cpp skips perf counters
|
||||
if let Ok(f) = std::panic::catch_unwind(|| {}) {
|
||||
let _ = f;
|
||||
}
|
||||
(self.lib.set_warmup)(self.ctx.ptr, true);
|
||||
// set_warmup may not exist in all builds; call decode with 1 token
|
||||
let dummy_token: llama_sys::llama_token = 1; // BOS-ish token
|
||||
@@ -279,7 +275,11 @@ impl InferenceEngine {
|
||||
}
|
||||
|
||||
let memory = (self.lib.get_memory)(self.ctx.ptr);
|
||||
(self.lib.memory_clear)(memory, false);
|
||||
if memory.is_null() {
|
||||
warn!("llama_get_memory returned NULL during warmup — KV cache not cleared");
|
||||
} else {
|
||||
(self.lib.memory_clear)(memory, false);
|
||||
}
|
||||
}
|
||||
|
||||
info!("Warmup complete");
|
||||
@@ -295,6 +295,21 @@ impl InferenceEngine {
|
||||
prompt: &str,
|
||||
sampler_cfg: SamplerConfig,
|
||||
) -> Result<GenerationSession> {
|
||||
// Clear KV cache from any previous session so the new prefill starts at position 0.
|
||||
// `true` forces a full clear including M-RoPE position metadata; `false` only clears
|
||||
// cell occupancy flags and leaves position data intact, which causes the
|
||||
// "inconsistent sequence positions" error on models that use M-RoPE (e.g. Qwen).
|
||||
unsafe {
|
||||
let memory = (self.lib.get_memory)(self.ctx.ptr);
|
||||
if memory.is_null() {
|
||||
tracing::error!("llama_get_memory returned NULL — cannot clear KV cache, refusing to create session to avoid decode corruption");
|
||||
anyhow::bail!("KV cache clear failed: llama_get_memory returned NULL");
|
||||
} else {
|
||||
(self.lib.memory_clear)(memory, true);
|
||||
tracing::info!("engine.kv_cache_cleared");
|
||||
}
|
||||
}
|
||||
|
||||
let vocab = unsafe { (self.lib.model_get_vocab)(self._model.ptr) };
|
||||
GenerationSession::with_vocab(
|
||||
Arc::clone(&self.lib),
|
||||
|
||||
Reference in new issue
Block a user