evandro/ForgeLM
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

perf: switch to qwen-style chat template

Browse Source
This commit is contained in:
Evandro Fonseca Junior committed 2026-03-26 17:49:17 -03:00
1 parent f84570afe4
commit 305c74e264
1 file changed
+79 -44
Download Patch File Download Diff File Expand all files Collapse all files
crates/inference-server/src/compat.rs
+79 -44
View File
@@ -301,9 +301,7 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
.any(|message| matches!(message.role, Role::System | Role::Developer));
if !has_system {
prompt.push_str("System: ");
prompt.push_str(&config.default_system_prompt);
prompt.push('\n');
append_chat_message(&mut prompt, "system", None, &config.default_system_prompt);
}
if let Some(format) = &req.response_format {
@@ -316,37 +314,31 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
for msg in &req.messages {
let role = match msg.role {
Role::System => "System",
Role::Developer => "Developer",
Role::User => "User",
Role::Assistant => "Assistant",
Role::Tool => "Tool",
Role::System => "system",
Role::Developer => "system",
Role::User => "user",
Role::Assistant => "assistant",
Role::Tool => "tool",
};
prompt.push_str(role);
if let Some(name) = &msg.name {
prompt.push_str("(");
prompt.push_str(name);
prompt.push(')');
}
prompt.push_str(": ");
if let Some(tool_calls) = &msg.tool_calls {
let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned());
prompt.push_str(&tool_json);
let content = if let Some(tool_calls) = &msg.tool_calls {
serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned())
} else {
prompt.push_str(msg.text());
}
msg.text().to_owned()
};
if let Some(tool_call_id) = &msg.tool_call_id {
prompt.push_str(" [tool_call_id=");
prompt.push_str(tool_call_id);
prompt.push(']');
let mut with_id = content;
with_id.push_str("\n[tool_call_id=");
with_id.push_str(tool_call_id);
with_id.push(']');
append_chat_message(&mut prompt, role, msg.name.as_deref(), &with_id);
} else {
append_chat_message(&mut prompt, role, msg.name.as_deref(), &content);
}
prompt.push('\n');
}
prompt.push_str("Assistant:");
prompt.push_str("<|im_start|>assistant\n");
PreparedPrompt {
prompt,
@@ -398,14 +390,20 @@ pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec<DeltaToolCall> {
fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) {
match format.format_type.as_str() {
"json_object" => {
prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n");
append_chat_message(
prompt,
"system",
None,
"Return valid JSON only. Do not wrap it in markdown fences.",
);
}
"json_schema" => {
prompt.push_str("System: Return valid JSON only and follow this schema exactly: ");
let mut instruction =
"Return valid JSON only and follow this schema exactly: ".to_owned();
if let Some(schema) = &format.json_schema {
prompt.push_str(&schema.to_string());
instruction.push_str(&schema.to_string());
}
prompt.push('\n');
append_chat_message(prompt, "system", None, &instruction);
}
_ => {}
}
@@ -427,32 +425,67 @@ fn append_tool_instructions(
})
.collect();
prompt.push_str("System: Available tools: ");
prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned()));
prompt.push('\n');
prompt
.push_str("System: If you need a tool, respond with JSON only using one of these shapes: ");
prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#);
prompt.push_str(" or ");
prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#);
prompt.push_str(". If no tool is needed, answer normally.\n");
append_chat_message(
prompt,
"system",
None,
&format!(
"Available tools: {}",
serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())
),
);
append_chat_message(
prompt,
"system",
None,
&format!(
"If you need a tool, respond with JSON only using one of these shapes: {} or {}. If no tool is needed, answer normally.",
r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#,
r#"{"name":"tool_name","arguments":{}}"#,
),
);
match tool_choice {
Some(ToolChoice::Mode(mode)) if mode == "required" => {
prompt.push_str("System: You must call a tool before replying.\n");
append_chat_message(
prompt,
"system",
None,
"You must call a tool before replying.",
);
}
Some(ToolChoice::Mode(mode)) if mode == "none" => {
prompt.push_str("System: Do not call tools in this response.\n");
append_chat_message(
prompt,
"system",
None,
"Do not call tools in this response.",
);
}
Some(ToolChoice::Specific(choice)) => {
prompt.push_str("System: You must call the tool named ");
prompt.push_str(&choice.function.name);
prompt.push_str(".\n");
append_chat_message(
prompt,
"system",
None,
&format!("You must call the tool named {}.", choice.function.name),
);
}
_ => {}
}
}
fn append_chat_message(prompt: &mut String, role: &str, name: Option<&str>, content: &str) {
prompt.push_str("<|im_start|>");
prompt.push_str(role);
if let Some(name) = name {
prompt.push(' ');
prompt.push_str(name);
}
prompt.push('\n');
prompt.push_str(content);
prompt.push_str("<|im_end|>\n");
}
fn parse_tool_calls(raw_output: &str) -> Option<Vec<ToolCall>> {
let trimmed = raw_output.trim();
if trimmed.is_empty() {
@@ -661,6 +694,8 @@ mod tests {
assert!(prepared.prompt.contains("Default coding system prompt"));
assert!(prepared.prompt.contains("list_files"));
assert!(prepared.prompt.contains("must call a tool"));
assert!(prepared.prompt.contains("<|im_start|>system\n"));
assert!(prepared.prompt.ends_with("<|im_start|>assistant\n"));
}
#[test]