fix(quick-1): bail on NULL KV cache and improve prefill error diagnostics

- engine.rs: new_session() now returns Err when llama_get_memory returns NULL
  instead of silently continuing, preventing decode corruption on 2nd request
- session.rs: decode_prefill error includes token count for easier diagnosis
This commit is contained in:
Evandro Fonseca Junior committed 2026-03-30 17:28:39 -03:00
1 parent 54a58ee9c7
commit d9198795bf
2 files changed
+45 -27

No files matched your search

+25 -22
View File
@@ -52,18 +52,6 @@ struct PrefillChunkPlan {
unsafe impl Send for GenerationSession {}
impl GenerationSession {
/// Tokenize prompt and run prefill decode. Returns a session ready to generate.
pub fn new(
_lib: Arc<LlamaLib>,
_ctx: *mut llama_context,
_sampler_cfg: SamplerConfig,
) -> Result<Self> {
// Get vocab from context's model (via a workaround: we pass vocab separately)
// For now we'll get it indirectly — see Engine::new_session which passes vocab.
// This constructor signature will be refined; use the vocab variant below.
bail!("Use GenerationSession::with_vocab() instead")
}
/// Full constructor called by InferenceEngine::new_session().
pub fn with_vocab(
lib: Arc<LlamaLib>,
@@ -159,8 +147,10 @@ impl GenerationSession {
let prefill_ms = prefill_start.elapsed().as_secs_f64() * 1000.0;
let prefill_tps = n_prompt_tokens as f64 / (prefill_ms / 1000.0);
info!(
"Prefill finished in {:.2} ms ({:.2} t/s)",
prefill_ms, prefill_tps
n_prompt_tokens,
prefill_ms = format_args!("{:.2}", prefill_ms),
prefill_tps = format_args!("{:.2}", prefill_tps),
"session.prefill_complete"
);
// create session state
@@ -224,6 +214,14 @@ impl GenerationSession {
self.n_decoded += 1;
self.tokens_generated += 1;
if self.tokens_generated % 100 == 0 {
tracing::debug!(
tokens_generated = self.tokens_generated,
max_tokens = self.max_tokens,
"session.generation_progress"
);
}
// Detokenize
let token_text = self.detokenize_token(new_token);
@@ -249,14 +247,18 @@ impl GenerationSession {
self.tokens_per_sec = self.tokens_generated as f64 / elapsed;
}
}
let stop_reason = if self.tokens_generated >= self.max_tokens {
"max_tokens"
} else {
"eos_or_stop_sequence"
};
info!(
tokens_generated = self.tokens_generated,
tps = format!("{:.1}", self.tokens_per_sec),
prefill_ms = self
.prefill_ms
.map(|v| format!("{:.1}", v))
.unwrap_or_default(),
"Generation complete"
max_tokens = self.max_tokens,
tps = self.tokens_per_sec,
prefill_ms = self.prefill_ms.unwrap_or(0.0),
stop_reason,
"session.generation_complete"
);
}
@@ -347,9 +349,10 @@ fn decode_prefill(
if ret != 0 {
bail!(
"llama_decode (prefill) returned {} while decoding prompt chunk {:?}",
"llama_decode (prefill) returned {} for chunk {:?} — likely KV cache not cleared or context window ({} tokens) exceeded by prompt",
ret,
plan.range
plan.range,
tokens.len()
);
}
}