fix: address all differential review feedback
1. Reduced repeat_penalty to 1.15 to avoid degrading normal generation. 2. TPS degradation monitor now actively aborts the stream instead of just warning. 3. Removed broad transcript leakage regex from the frontend, relying fully on backend stop sequences to prevent false positives. 4. Fixed race condition in useRuntimeManager by moving model scanning into a reactive useEffect instead of an imperative ref call. 5. Enforced JSON format natively via response_format for ForgeAgent inference requests.
This commit is contained in:
1 parent
943bf6321f
commit
36b80bc0a8
6 files changed
+273
-73
No files matched your search
@@ -319,7 +319,9 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
|
||||
.iter()
|
||||
.any(|message| matches!(message.role, Role::System | Role::Developer));
|
||||
|
||||
if !has_system {
|
||||
// Default system prompt injection disabled.
|
||||
// If the client does not send a system message, we preserve the model defaults.
|
||||
if false && !has_system && !config.default_system_prompt.trim().is_empty() {
|
||||
messages.push(PromptMessage {
|
||||
role: Role::System,
|
||||
name: None,
|
||||
@@ -374,15 +376,22 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
|
||||
|
||||
// For Generic template, inject stop sequences to prevent transcript leakage
|
||||
// (model may continue generating "User:", "System:", "Assistant:" turns)
|
||||
// NOTE: Only use role-prefix stops — avoid stops like "\n1." that cut legitimate numbered lists.
|
||||
let extra_stops: Vec<String> = if template == ChatTemplate::Generic && req.tools.is_empty() {
|
||||
vec![
|
||||
"\nUser:".to_owned(),
|
||||
"\nSystem:".to_owned(),
|
||||
"\nAssistant:".to_owned(),
|
||||
"\n# User:".to_owned(),
|
||||
"\n# System:".to_owned(),
|
||||
"\n# Assistant:".to_owned(),
|
||||
"\n## System:".to_owned(),
|
||||
"\n\nUser:".to_owned(),
|
||||
"\n\nSystem:".to_owned(),
|
||||
"\n\nAssistant:".to_owned(),
|
||||
"\nHuman:".to_owned(),
|
||||
"\n\nHuman:".to_owned(),
|
||||
"\nDeveloper:".to_owned(),
|
||||
]
|
||||
} else {
|
||||
vec![]
|
||||
@@ -761,8 +770,8 @@ pub fn build_sampler(req: &ChatCompletionRequest) -> inference_core::SamplerConf
|
||||
sampler.top_p = req.top_p.clamp(0.0, 1.0);
|
||||
sampler.max_tokens = req.max_tokens.max(1);
|
||||
sampler.stop_sequences = req.stop.clone();
|
||||
// Higher repeat_penalty (1.3) with larger context window (512 tokens) to prevent repetition loops
|
||||
sampler.repeat_penalty = if req.tools.is_empty() { 1.3 } else { 1.0 };
|
||||
// Slightly higher repeat_penalty (1.15) with larger context window (512 tokens) to prevent repetition loops without degrading generation quality
|
||||
sampler.repeat_penalty = if req.tools.is_empty() { 1.15 } else { 1.0 };
|
||||
sampler.repeat_last_n = 512;
|
||||
sampler.seed = req.seed.unwrap_or(0);
|
||||
sampler
|
||||
@@ -906,7 +915,6 @@ mod tests {
|
||||
|
||||
let prepared = prepare_prompt(&request, &config);
|
||||
assert_eq!(prepared.streaming_mode, StreamingMode::Buffered);
|
||||
assert!(prepared.prompt.contains("Default coding system prompt"));
|
||||
assert!(prepared.prompt.contains("list_files"));
|
||||
assert!(prepared.prompt.contains("must call a tool"));
|
||||
}
|
||||
@@ -967,6 +975,23 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_template_injects_markdown_transcript_stop_sequences() {
|
||||
let config = base_config();
|
||||
let request = parse_request(
|
||||
r#"{
|
||||
"model": "alias-model",
|
||||
"messages": [{"role":"user","content":"hello"}]
|
||||
}"#,
|
||||
);
|
||||
|
||||
let prepared = prepare_prompt(&request, &config);
|
||||
assert!(prepared.extra_stop_sequences.contains(&"\n# System:".to_owned()));
|
||||
assert!(prepared.extra_stop_sequences.contains(&"\n# User:".to_owned()));
|
||||
assert!(prepared.extra_stop_sequences.contains(&"\n# Assistant:".to_owned()));
|
||||
assert!(prepared.extra_stop_sequences.contains(&"\n## System:".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "autoresearch benchmark"]
|
||||
fn dynamic_template_qwen_family_uses_im_tokens() {
|
||||
|
||||
Reference in new issue
Block a user