fix(quick-1): bail on NULL KV cache and improve prefill error diagnostics

- engine.rs: new_session() now returns Err when llama_get_memory returns NULL
  instead of silently continuing, preventing decode corruption on 2nd request
- session.rs: decode_prefill error includes token count for easier diagnosis
This commit is contained in:
Evandro Fonseca Junior committed 2026-03-30 17:28:39 -03:00
1 parent 54a58ee9c7
commit d9198795bf
2 files changed
+45 -27

No files matched your search

+20 -5
View File
@@ -263,10 +263,6 @@ impl InferenceEngine {
info!("Running engine warmup (dummy decode)…");
unsafe {
// Mark context as warmup so llama.cpp skips perf counters
if let Ok(f) = std::panic::catch_unwind(|| {}) {
let _ = f;
}
(self.lib.set_warmup)(self.ctx.ptr, true);
// set_warmup may not exist in all builds; call decode with 1 token
let dummy_token: llama_sys::llama_token = 1; // BOS-ish token
@@ -279,7 +275,11 @@ impl InferenceEngine {
}
let memory = (self.lib.get_memory)(self.ctx.ptr);
(self.lib.memory_clear)(memory, false);
if memory.is_null() {
warn!("llama_get_memory returned NULL during warmup — KV cache not cleared");
} else {
(self.lib.memory_clear)(memory, false);
}
}
info!("Warmup complete");
@@ -295,6 +295,21 @@ impl InferenceEngine {
prompt: &str,
sampler_cfg: SamplerConfig,
) -> Result<GenerationSession> {
// Clear KV cache from any previous session so the new prefill starts at position 0.
// `true` forces a full clear including M-RoPE position metadata; `false` only clears
// cell occupancy flags and leaves position data intact, which causes the
// "inconsistent sequence positions" error on models that use M-RoPE (e.g. Qwen).
unsafe {
let memory = (self.lib.get_memory)(self.ctx.ptr);
if memory.is_null() {
tracing::error!("llama_get_memory returned NULL — cannot clear KV cache, refusing to create session to avoid decode corruption");
anyhow::bail!("KV cache clear failed: llama_get_memory returned NULL");
} else {
(self.lib.memory_clear)(memory, true);
tracing::info!("engine.kv_cache_cleared");
}
}
let vocab = unsafe { (self.lib.model_get_vocab)(self._model.ptr) };
GenerationSession::with_vocab(
Arc::clone(&self.lib),