feat: add family-based chat templates

autoresearch-dashboard.md

@@ -1,14 +1,9 @@
-# Autoresearch Dashboard: latency-p95-chat-completions
+# Autoresearch Dashboard: dynamic-chat-templates
 
-**Runs:** 6 | **Kept:** 2 | **Discarded:** 4 | **Crashed:** 0
+**Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0
-**Baseline:** p95_ms: 941.252ms (#1)
+**Baseline:** template_score: 20.0% (#7)
-**Best:** p95_ms: 925.587ms (#6, -1.7%)
+**Best:** template_score: 20.0% (#7, +0.0%)
 
-| # | commit | p95_ms | status | description |
+| # | commit | template_score | status | description |
-|---|--------|--------|--------|-------------|
+|---|--------|----------------|--------|-------------|
-| 1 | ceed0e5 | 941.252ms (+0.0%) | keep | baseline |
+| 7 | 89ef220 | 20.0% (+0.0%) | keep | baseline |
-| 2 | 5269f43 | 1032.315ms (+9.7%) | discard | use stack buffer for token detokenization |
-| 3 | 994e607 | 1004.435ms (+6.7%) | discard | slim turbo4b runtime profile |
-| 4 | db92212 | 3027.145ms (+221.6%) | discard | switch to qwen-style chat template |
-| 5 | caf4654 | 965.663ms (+2.6%) | discard | stop generation on default turn boundaries |
-| 6 | e8402d2 | 925.587ms (-1.7%) | keep | disable llama perf counters in hot path |

autoresearch.jsonl

@@ -5,3 +5,5 @@
 {"run":4,"commit":"db92212","metric":3027.145,"metrics":{"p50_ms":974.937,"mean_ms":1467.026,"completion_tps_mean":41.360,"success_count":7},"status":"discard","description":"switch to qwen-style chat template","timestamp":1774558217,"segment":0}
 {"run":5,"commit":"caf4654","metric":965.663,"metrics":{"p50_ms":877.030,"mean_ms":897.251,"completion_tps_mean":53.686,"success_count":7},"status":"discard","description":"stop generation on default turn boundaries","timestamp":1774558444,"segment":0}
 {"run":6,"commit":"e8402d2","metric":925.587,"metrics":{"p50_ms":893.913,"mean_ms":899.895,"completion_tps_mean":53.618,"success_count":7},"status":"keep","description":"disable llama perf counters in hot path","timestamp":1774558642,"segment":0}
+{"type":"config","name":"dynamic-chat-templates","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
+{"run":7,"commit":"89ef220","metric":20.0,"metrics":{"passed_cases":1,"failed_cases":4,"total_cases":5},"status":"keep","description":"baseline","timestamp":1774559683,"segment":1}

crates/inference-server/src/compat.rs

@@ -191,6 +191,15 @@ pub struct FinalizedAssistant {
     pub finish_reason: FinishReason,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ChatTemplate {
+    Generic,
+    Qwen,
+    Gemma,
+    Llama,
+    Kimi,
+}
+
 pub fn authorize_request(
     headers: &HeaderMap,
     config: &ServerConfig,
@@ -294,6 +303,7 @@ pub fn validate_request(
 }
 
 pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> PreparedPrompt {
+    let template = select_chat_template(req, config);
     let mut prompt = String::with_capacity(1024);
     let has_system = req
         .messages
@@ -301,52 +311,46 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre
         .any(|message| matches!(message.role, Role::System | Role::Developer));
 
     if !has_system {
-        prompt.push_str("System: ");
+        append_chat_message(
-        prompt.push_str(&config.default_system_prompt);
+            &mut prompt,
-        prompt.push('\n');
+            template,
+            Role::System,
+            None,
+            &config.default_system_prompt,
+        );
     }
 
     if let Some(format) = &req.response_format {
-        append_response_format_instruction(&mut prompt, format);
+        append_response_format_instruction(&mut prompt, template, format);
     }
 
     if !req.tools.is_empty() {
-        append_tool_instructions(&mut prompt, &req.tools, req.tool_choice.as_ref());
+        append_tool_instructions(&mut prompt, template, &req.tools, req.tool_choice.as_ref());
     }
 
     for msg in &req.messages {
-        let role = match msg.role {
+        let mut content = if let Some(tool_calls) = &msg.tool_calls {
-            Role::System => "System",
+            serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned())
-            Role::Developer => "Developer",
-            Role::User => "User",
-            Role::Assistant => "Assistant",
-            Role::Tool => "Tool",
-        };
-        prompt.push_str(role);
-        if let Some(name) = &msg.name {
-            prompt.push_str("(");
-            prompt.push_str(name);
-            prompt.push(')');
-        }
-        prompt.push_str(": ");
-
-        if let Some(tool_calls) = &msg.tool_calls {
-            let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned());
-            prompt.push_str(&tool_json);
         } else {
-            prompt.push_str(msg.text());
+            msg.text().to_owned()
-        }
+        };
 
         if let Some(tool_call_id) = &msg.tool_call_id {
-            prompt.push_str(" [tool_call_id=");
+            content.push_str(" [tool_call_id=");
-            prompt.push_str(tool_call_id);
+            content.push_str(tool_call_id);
-            prompt.push(']');
+            content.push(']');
         }
 
-        prompt.push('\n');
+        append_chat_message(
+            &mut prompt,
+            template,
+            msg.role.clone(),
+            msg.name.as_deref(),
+            &content,
+        );
     }
 
-    prompt.push_str("Assistant:");
+    append_assistant_prefix(&mut prompt, template);
 
     PreparedPrompt {
         prompt,
@@ -395,17 +399,47 @@ pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec<DeltaToolCall> {
         .collect()
 }
 
-fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) {
+fn select_chat_template(req: &ChatCompletionRequest, config: &ServerConfig) -> ChatTemplate {
+    let model_id = config
+        .resolve_model(&req.model)
+        .map(|model| model.root.as_str())
+        .unwrap_or(req.model.as_str());
+    let normalized = model_id.to_ascii_lowercase();
+
+    if normalized.contains("qwen") {
+        ChatTemplate::Qwen
+    } else if normalized.contains("gemma") {
+        ChatTemplate::Gemma
+    } else if normalized.contains("llama") {
+        ChatTemplate::Llama
+    } else if normalized.contains("kimi") {
+        ChatTemplate::Kimi
+    } else {
+        ChatTemplate::Generic
+    }
+}
+
+fn append_response_format_instruction(
+    prompt: &mut String,
+    template: ChatTemplate,
+    format: &ResponseFormat,
+) {
     match format.format_type.as_str() {
         "json_object" => {
-            prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n");
+            append_chat_message(
+                prompt,
+                template,
+                Role::System,
+                None,
+                "Return valid JSON only. Do not wrap it in markdown fences.",
+            );
         }
         "json_schema" => {
-            prompt.push_str("System: Return valid JSON only and follow this schema exactly: ");
+            let mut content = "Return valid JSON only and follow this schema exactly: ".to_owned();
             if let Some(schema) = &format.json_schema {
-                prompt.push_str(&schema.to_string());
+                content.push_str(&schema.to_string());
             }
-            prompt.push('\n');
+            append_chat_message(prompt, template, Role::System, None, &content);
         }
         _ => {}
     }
@@ -413,6 +447,7 @@ fn append_response_format_instruction(prompt: &mut String, format: &ResponseForm
 
 fn append_tool_instructions(
     prompt: &mut String,
+    template: ChatTemplate,
     tools: &[ToolDefinition],
     tool_choice: Option<&ToolChoice>,
 ) {
@@ -427,32 +462,150 @@ fn append_tool_instructions(
         })
         .collect();
 
-    prompt.push_str("System: Available tools: ");
+    let mut available_tools = "Available tools: ".to_owned();
-    prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned()));
+    available_tools
-    prompt.push('\n');
+        .push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned()));
-    prompt
+    append_chat_message(prompt, template, Role::System, None, &available_tools);
-        .push_str("System: If you need a tool, respond with JSON only using one of these shapes: ");
+
-    prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#);
+    let tool_usage = format!(
-    prompt.push_str(" or ");
+        "If you need a tool, respond with JSON only using one of these shapes: {} or {}. If no tool is needed, answer normally.",
-    prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#);
+        r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#,
-    prompt.push_str(". If no tool is needed, answer normally.\n");
+        r#"{"name":"tool_name","arguments":{}}"#
+    );
+    append_chat_message(prompt, template, Role::System, None, &tool_usage);
 
     match tool_choice {
         Some(ToolChoice::Mode(mode)) if mode == "required" => {
-            prompt.push_str("System: You must call a tool before replying.\n");
+            append_chat_message(
+                prompt,
+                template,
+                Role::System,
+                None,
+                "You must call a tool before replying.",
+            );
         }
         Some(ToolChoice::Mode(mode)) if mode == "none" => {
-            prompt.push_str("System: Do not call tools in this response.\n");
+            append_chat_message(
+                prompt,
+                template,
+                Role::System,
+                None,
+                "Do not call tools in this response.",
+            );
         }
         Some(ToolChoice::Specific(choice)) => {
-            prompt.push_str("System: You must call the tool named ");
+            let content = format!("You must call the tool named {}.", choice.function.name);
-            prompt.push_str(&choice.function.name);
+            append_chat_message(prompt, template, Role::System, None, &content);
-            prompt.push_str(".\n");
         }
         _ => {}
     }
 }
 
+fn append_chat_message(
+    prompt: &mut String,
+    template: ChatTemplate,
+    role: Role,
+    name: Option<&str>,
+    content: &str,
+) {
+    let content = if let Some(name) = name {
+        format!("{} ({})", content, name)
+    } else {
+        content.to_owned()
+    };
+
+    match template {
+        ChatTemplate::Generic => {
+            prompt.push_str(generic_role_name(&role));
+            prompt.push_str(": ");
+            prompt.push_str(&content);
+            prompt.push('\n');
+        }
+        ChatTemplate::Qwen => {
+            prompt.push_str("<|im_start|>");
+            prompt.push_str(qwen_role_name(&role));
+            prompt.push('\n');
+            prompt.push_str(&content);
+            prompt.push_str("<|im_end|>\n");
+        }
+        ChatTemplate::Gemma => {
+            prompt.push_str("<start_of_turn>");
+            prompt.push_str(gemma_role_name(&role));
+            prompt.push('\n');
+            prompt.push_str(&content);
+            prompt.push_str("<end_of_turn>\n");
+        }
+        ChatTemplate::Llama => match role {
+            Role::Assistant => {
+                prompt.push_str(&content);
+                prompt.push('\n');
+            }
+            _ => {
+                prompt.push_str("[INST] ");
+                if matches!(role, Role::System | Role::Developer) {
+                    prompt.push_str("<<SYS>>\n");
+                    prompt.push_str(&content);
+                    prompt.push_str("\n<</SYS>>");
+                } else {
+                    prompt.push_str(&content);
+                }
+                prompt.push_str(" [/INST]\n");
+            }
+        },
+        ChatTemplate::Kimi => {
+            prompt.push_str(kimi_role_marker(&role));
+            prompt.push('\n');
+            prompt.push_str(&content);
+            prompt.push('\n');
+        }
+    }
+}
+
+fn append_assistant_prefix(prompt: &mut String, template: ChatTemplate) {
+    match template {
+        ChatTemplate::Generic => prompt.push_str("Assistant:"),
+        ChatTemplate::Qwen => prompt.push_str("<|im_start|>assistant\n"),
+        ChatTemplate::Gemma => prompt.push_str("<start_of_turn>model\n"),
+        ChatTemplate::Llama => prompt.push_str("[INST] "),
+        ChatTemplate::Kimi => prompt.push_str("<|im_assistant|>\n"),
+    }
+}
+
+fn generic_role_name(role: &Role) -> &'static str {
+    match role {
+        Role::System => "System",
+        Role::Developer => "Developer",
+        Role::User => "User",
+        Role::Assistant => "Assistant",
+        Role::Tool => "Tool",
+    }
+}
+
+fn qwen_role_name(role: &Role) -> &'static str {
+    match role {
+        Role::System | Role::Developer => "system",
+        Role::User => "user",
+        Role::Assistant => "assistant",
+        Role::Tool => "tool",
+    }
+}
+
+fn gemma_role_name(role: &Role) -> &'static str {
+    match role {
+        Role::Assistant => "model",
+        Role::System | Role::Developer | Role::User | Role::Tool => "user",
+    }
+}
+
+fn kimi_role_marker(role: &Role) -> &'static str {
+    match role {
+        Role::System | Role::Developer => "<|im_system|>",
+        Role::User => "<|im_user|>",
+        Role::Assistant => "<|im_assistant|>",
+        Role::Tool => "<|im_tool|>",
+    }
+}
+
 fn parse_tool_calls(raw_output: &str) -> Option<Vec<ToolCall>> {
     let trimmed = raw_output.trim();
     if trimmed.is_empty() {

experiments/worklog.md

@@ -61,3 +61,15 @@
 - Result: p95_ms=3027.145, p50_ms=974.937, mean_ms=1467.026, completion_tps_mean=41.360, success_count=7.
 - Insight: The template did not make typical requests much faster and introduced severe tail latency, likely because some generations stopped much later or followed a worse decoding path for this server/model combination.
 - Next: revert this template change and focus on experiments that directly bound generation length or improve stop behavior without overhauling the prompt format.
+
+## Segment 1: Dynamic Chat Templates
+- New objective: add dynamic prompt templates by model family with built-in coverage for Qwen, Gemma, Llama, Kimi, and a generic fallback.
+- Benchmark driver: `./autoresearch.sh` now runs ignored template-specific tests and reports `template_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule for this segment is test-driven: higher `template_score` wins, while the regular non-ignored test suite must remain green.
+
+### Run 7: baseline - template_score=20.0 (KEEP)
+- Timestamp: 2026-03-26 18:14
+- What changed: Re-initialized autoresearch to a new segment focused on dynamic chat templates, added a dedicated ignored test suite for Qwen, Gemma, Llama, Kimi, and generic fallback behavior, and switched the benchmark harness to score that suite.
+- Result: template_score=20.0, passed_cases=1, failed_cases=4, total_cases=5.
+- Insight: Only the generic fallback works today; all named family-template expectations fail because prompt rendering is still generic and model-agnostic.
+- Next: add a minimal model-family detector and template registry in `crates/inference-server/src/compat.rs`, then route `prepare_prompt()` through it.