perf: switch to qwen-style chat template

crates/inference-server/src/compat.rs

@@ -301,9 +301,7 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
         .any(|message| matches!(message.role, Role::System | Role::Developer));
 
     if !has_system {
-        prompt.push_str("System: ");
+        append_chat_message(&mut prompt, "system", None, &config.default_system_prompt);
-        prompt.push_str(&config.default_system_prompt);
-        prompt.push('\n');
     }
 
     if let Some(format) = &req.response_format {
@@ -316,37 +314,31 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
 
     for msg in &req.messages {
         let role = match msg.role {
-            Role::System => "System",
+            Role::System => "system",
-            Role::Developer => "Developer",
+            Role::Developer => "system",
-            Role::User => "User",
+            Role::User => "user",
-            Role::Assistant => "Assistant",
+            Role::Assistant => "assistant",
-            Role::Tool => "Tool",
+            Role::Tool => "tool",
         };
-        prompt.push_str(role);
-        if let Some(name) = &msg.name {
-            prompt.push_str("(");
-            prompt.push_str(name);
-            prompt.push(')');
-        }
-        prompt.push_str(": ");
 
-        if let Some(tool_calls) = &msg.tool_calls {
+        let content = if let Some(tool_calls) = &msg.tool_calls {
-            let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned());
+            serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned())
-            prompt.push_str(&tool_json);
         } else {
-            prompt.push_str(msg.text());
+            msg.text().to_owned()
-        }
+        };
 
         if let Some(tool_call_id) = &msg.tool_call_id {
-            prompt.push_str(" [tool_call_id=");
+            let mut with_id = content;
-            prompt.push_str(tool_call_id);
+            with_id.push_str("\n[tool_call_id=");
-            prompt.push(']');
+            with_id.push_str(tool_call_id);
+            with_id.push(']');
+            append_chat_message(&mut prompt, role, msg.name.as_deref(), &with_id);
+        } else {
+            append_chat_message(&mut prompt, role, msg.name.as_deref(), &content);
         }
-
-        prompt.push('\n');
     }
 
-    prompt.push_str("Assistant:");
+    prompt.push_str("<|im_start|>assistant\n");
 
     PreparedPrompt {
         prompt,
@@ -398,14 +390,20 @@ pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec<DeltaToolCall> {
 fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) {
     match format.format_type.as_str() {
         "json_object" => {
-            prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n");
+            append_chat_message(
+                prompt,
+                "system",
+                None,
+                "Return valid JSON only. Do not wrap it in markdown fences.",
+            );
         }
         "json_schema" => {
-            prompt.push_str("System: Return valid JSON only and follow this schema exactly: ");
+            let mut instruction =
+                "Return valid JSON only and follow this schema exactly: ".to_owned();
             if let Some(schema) = &format.json_schema {
-                prompt.push_str(&schema.to_string());
+                instruction.push_str(&schema.to_string());
             }
-            prompt.push('\n');
+            append_chat_message(prompt, "system", None, &instruction);
         }
         _ => {}
     }
@@ -427,32 +425,67 @@ fn append_tool_instructions(
         })
         .collect();
 
-    prompt.push_str("System: Available tools: ");
+    append_chat_message(
-    prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned()));
+        prompt,
-    prompt.push('\n');
+        "system",
-    prompt
+        None,
-        .push_str("System: If you need a tool, respond with JSON only using one of these shapes: ");
+        &format!(
-    prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#);
+            "Available tools: {}",
-    prompt.push_str(" or ");
+            serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())
-    prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#);
+        ),
-    prompt.push_str(". If no tool is needed, answer normally.\n");
+    );
+    append_chat_message(
+        prompt,
+        "system",
+        None,
+        &format!(
+            "If you need a tool, respond with JSON only using one of these shapes: {} or {}. If no tool is needed, answer normally.",
+            r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#,
+            r#"{"name":"tool_name","arguments":{}}"#,
+        ),
+    );
 
     match tool_choice {
         Some(ToolChoice::Mode(mode)) if mode == "required" => {
-            prompt.push_str("System: You must call a tool before replying.\n");
+            append_chat_message(
+                prompt,
+                "system",
+                None,
+                "You must call a tool before replying.",
+            );
         }
         Some(ToolChoice::Mode(mode)) if mode == "none" => {
-            prompt.push_str("System: Do not call tools in this response.\n");
+            append_chat_message(
+                prompt,
+                "system",
+                None,
+                "Do not call tools in this response.",
+            );
         }
         Some(ToolChoice::Specific(choice)) => {
-            prompt.push_str("System: You must call the tool named ");
+            append_chat_message(
-            prompt.push_str(&choice.function.name);
+                prompt,
-            prompt.push_str(".\n");
+                "system",
+                None,
+                &format!("You must call the tool named {}.", choice.function.name),
+            );
         }
         _ => {}
     }
 }
 
+fn append_chat_message(prompt: &mut String, role: &str, name: Option<&str>, content: &str) {
+    prompt.push_str("<|im_start|>");
+    prompt.push_str(role);
+    if let Some(name) = name {
+        prompt.push(' ');
+        prompt.push_str(name);
+    }
+    prompt.push('\n');
+    prompt.push_str(content);
+    prompt.push_str("<|im_end|>\n");
+}
+
 fn parse_tool_calls(raw_output: &str) -> Option<Vec<ToolCall>> {
     let trimmed = raw_output.trim();
     if trimmed.is_empty() {
@@ -661,6 +694,8 @@ mod tests {
         assert!(prepared.prompt.contains("Default coding system prompt"));
         assert!(prepared.prompt.contains("list_files"));
         assert!(prepared.prompt.contains("must call a tool"));
+        assert!(prepared.prompt.contains("<|im_start|>system\n"));
+        assert!(prepared.prompt.ends_with("<|im_start|>assistant\n"));
     }
 
     #[test]