fix: stop transcript leakage and repetition loops in Generic chat template
Backend (inference-server/compat.rs): - PreparedPrompt now carries extra_stop_sequences injected by template - Generic template injects User:/System:/Assistant:/Human: stop sequences to prevent the model from continuing the transcript after its reply - Increased repeat_penalty 1.15→1.3 and repeat_last_n 256→512 to break repetition loops more aggressively Backend (inference-server/routes.rs): - Merge template extra_stop_sequences into sampler before inference Frontend (chat.ts): - Added 'System:' and 'Human:' to CHAT_TRANSCRIPT_STOP_SEQUENCES - Extended TRANSCRIPT_LEAKAGE_PATTERN to include 'Human:' Frontend (ChatView.tsx): - buildApiMessages now strips leaked transcript from assistant messages before sending history back to the model - Reduced history window to 20 messages to limit context pollution
This commit is contained in:
1 parent
5f86c69b03
commit
88f4319c3f
6 files changed
+1951
-7
No files matched your search
@@ -176,6 +176,8 @@ pub struct ModelDescriptor {
|
||||
pub struct PreparedPrompt {
|
||||
pub prompt: String,
|
||||
pub streaming_mode: StreamingMode,
|
||||
/// Extra stop sequences injected by the template (e.g. Generic transcript stops)
|
||||
pub extra_stop_sequences: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -370,6 +372,22 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
|
||||
append_assistant_prefix(&mut prompt, template);
|
||||
}
|
||||
|
||||
// For Generic template, inject stop sequences to prevent transcript leakage
|
||||
// (model may continue generating "User:", "System:", "Assistant:" turns)
|
||||
let extra_stops: Vec<String> = if template == ChatTemplate::Generic && req.tools.is_empty() {
|
||||
vec![
|
||||
"\nUser:".to_owned(),
|
||||
"\nSystem:".to_owned(),
|
||||
"\nAssistant:".to_owned(),
|
||||
"\n\nUser:".to_owned(),
|
||||
"\n\nSystem:".to_owned(),
|
||||
"\n\nAssistant:".to_owned(),
|
||||
"\nHuman:".to_owned(),
|
||||
]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
PreparedPrompt {
|
||||
prompt,
|
||||
streaming_mode: if req.tools.is_empty() {
|
||||
@@ -377,6 +395,7 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
|
||||
} else {
|
||||
StreamingMode::Buffered
|
||||
},
|
||||
extra_stop_sequences: extra_stops,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -742,9 +761,9 @@ pub fn build_sampler(req: &ChatCompletionRequest) -> inference_core::SamplerConf
|
||||
sampler.top_p = req.top_p.clamp(0.0, 1.0);
|
||||
sampler.max_tokens = req.max_tokens.max(1);
|
||||
sampler.stop_sequences = req.stop.clone();
|
||||
// Higher repeat_penalty (1.15) with larger context window (256 tokens) to prevent repetition loops
|
||||
sampler.repeat_penalty = if req.tools.is_empty() { 1.15 } else { 1.0 };
|
||||
sampler.repeat_last_n = 256;
|
||||
// Higher repeat_penalty (1.3) with larger context window (512 tokens) to prevent repetition loops
|
||||
sampler.repeat_penalty = if req.tools.is_empty() { 1.3 } else { 1.0 };
|
||||
sampler.repeat_last_n = 512;
|
||||
sampler.seed = req.seed.unwrap_or(0);
|
||||
sampler
|
||||
}
|
||||
|
||||
@@ -167,10 +167,21 @@ async fn chat_completions_handler(
|
||||
);
|
||||
}
|
||||
|
||||
let sampler = build_sampler(&req);
|
||||
let mut sampler = build_sampler(&req);
|
||||
let prepared = prepare_prompt(&req, &state.server_config);
|
||||
tracing::debug!(prompt_len = prepared.prompt.len(), "chat_completions: prompt prepared");
|
||||
|
||||
// Merge template-injected stop sequences with request-provided ones
|
||||
if !prepared.extra_stop_sequences.is_empty() {
|
||||
let mut stops = sampler.stop_sequences.clone();
|
||||
for s in &prepared.extra_stop_sequences {
|
||||
if !stops.contains(s) {
|
||||
stops.push(s.clone());
|
||||
}
|
||||
}
|
||||
sampler.stop_sequences = stops;
|
||||
}
|
||||
|
||||
if req.stream {
|
||||
tracing::info!("chat_completions: starting stream response");
|
||||
stream_response(
|
||||
|
||||
Reference in new issue
Block a user