From 305c74e264d2c1306f33e714fde4c86523a00b61 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 17:49:17 -0300 Subject: [PATCH] perf: switch to qwen-style chat template --- crates/inference-server/src/compat.rs | 123 +++++++++++++++++--------- 1 file changed, 79 insertions(+), 44 deletions(-) diff --git a/crates/inference-server/src/compat.rs b/crates/inference-server/src/compat.rs index 0adbc6d..7891181 100644 --- a/crates/inference-server/src/compat.rs +++ b/crates/inference-server/src/compat.rs @@ -301,9 +301,7 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre .any(|message| matches!(message.role, Role::System | Role::Developer)); if !has_system { - prompt.push_str("System: "); - prompt.push_str(&config.default_system_prompt); - prompt.push('\n'); + append_chat_message(&mut prompt, "system", None, &config.default_system_prompt); } if let Some(format) = &req.response_format { @@ -316,37 +314,31 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre for msg in &req.messages { let role = match msg.role { - Role::System => "System", - Role::Developer => "Developer", - Role::User => "User", - Role::Assistant => "Assistant", - Role::Tool => "Tool", + Role::System => "system", + Role::Developer => "system", + Role::User => "user", + Role::Assistant => "assistant", + Role::Tool => "tool", }; - prompt.push_str(role); - if let Some(name) = &msg.name { - prompt.push_str("("); - prompt.push_str(name); - prompt.push(')'); - } - prompt.push_str(": "); - if let Some(tool_calls) = &msg.tool_calls { - let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned()); - prompt.push_str(&tool_json); + let content = if let Some(tool_calls) = &msg.tool_calls { + serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned()) } else { - prompt.push_str(msg.text()); - } + msg.text().to_owned() + }; if let Some(tool_call_id) = &msg.tool_call_id { - prompt.push_str(" [tool_call_id="); - prompt.push_str(tool_call_id); - prompt.push(']'); + let mut with_id = content; + with_id.push_str("\n[tool_call_id="); + with_id.push_str(tool_call_id); + with_id.push(']'); + append_chat_message(&mut prompt, role, msg.name.as_deref(), &with_id); + } else { + append_chat_message(&mut prompt, role, msg.name.as_deref(), &content); } - - prompt.push('\n'); } - prompt.push_str("Assistant:"); + prompt.push_str("<|im_start|>assistant\n"); PreparedPrompt { prompt, @@ -398,14 +390,20 @@ pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec { fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) { match format.format_type.as_str() { "json_object" => { - prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n"); + append_chat_message( + prompt, + "system", + None, + "Return valid JSON only. Do not wrap it in markdown fences.", + ); } "json_schema" => { - prompt.push_str("System: Return valid JSON only and follow this schema exactly: "); + let mut instruction = + "Return valid JSON only and follow this schema exactly: ".to_owned(); if let Some(schema) = &format.json_schema { - prompt.push_str(&schema.to_string()); + instruction.push_str(&schema.to_string()); } - prompt.push('\n'); + append_chat_message(prompt, "system", None, &instruction); } _ => {} } @@ -427,32 +425,67 @@ fn append_tool_instructions( }) .collect(); - prompt.push_str("System: Available tools: "); - prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())); - prompt.push('\n'); - prompt - .push_str("System: If you need a tool, respond with JSON only using one of these shapes: "); - prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#); - prompt.push_str(" or "); - prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#); - prompt.push_str(". If no tool is needed, answer normally.\n"); + append_chat_message( + prompt, + "system", + None, + &format!( + "Available tools: {}", + serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned()) + ), + ); + append_chat_message( + prompt, + "system", + None, + &format!( + "If you need a tool, respond with JSON only using one of these shapes: {} or {}. If no tool is needed, answer normally.", + r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#, + r#"{"name":"tool_name","arguments":{}}"#, + ), + ); match tool_choice { Some(ToolChoice::Mode(mode)) if mode == "required" => { - prompt.push_str("System: You must call a tool before replying.\n"); + append_chat_message( + prompt, + "system", + None, + "You must call a tool before replying.", + ); } Some(ToolChoice::Mode(mode)) if mode == "none" => { - prompt.push_str("System: Do not call tools in this response.\n"); + append_chat_message( + prompt, + "system", + None, + "Do not call tools in this response.", + ); } Some(ToolChoice::Specific(choice)) => { - prompt.push_str("System: You must call the tool named "); - prompt.push_str(&choice.function.name); - prompt.push_str(".\n"); + append_chat_message( + prompt, + "system", + None, + &format!("You must call the tool named {}.", choice.function.name), + ); } _ => {} } } +fn append_chat_message(prompt: &mut String, role: &str, name: Option<&str>, content: &str) { + prompt.push_str("<|im_start|>"); + prompt.push_str(role); + if let Some(name) = name { + prompt.push(' '); + prompt.push_str(name); + } + prompt.push('\n'); + prompt.push_str(content); + prompt.push_str("<|im_end|>\n"); +} + fn parse_tool_calls(raw_output: &str) -> Option> { let trimmed = raw_output.trim(); if trimmed.is_empty() { @@ -661,6 +694,8 @@ mod tests { assert!(prepared.prompt.contains("Default coding system prompt")); assert!(prepared.prompt.contains("list_files")); assert!(prepared.prompt.contains("must call a tool")); + assert!(prepared.prompt.contains("<|im_start|>system\n")); + assert!(prepared.prompt.ends_with("<|im_start|>assistant\n")); } #[test]