diff --git a/autoresearch-dashboard.md b/autoresearch-dashboard.md index 94ec57c..77b8e17 100644 --- a/autoresearch-dashboard.md +++ b/autoresearch-dashboard.md @@ -1,14 +1,9 @@ -# Autoresearch Dashboard: latency-p95-chat-completions +# Autoresearch Dashboard: dynamic-chat-templates -**Runs:** 6 | **Kept:** 2 | **Discarded:** 4 | **Crashed:** 0 -**Baseline:** p95_ms: 941.252ms (#1) -**Best:** p95_ms: 925.587ms (#6, -1.7%) +**Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0 +**Baseline:** template_score: 20.0% (#7) +**Best:** template_score: 20.0% (#7, +0.0%) -| # | commit | p95_ms | status | description | -|---|--------|--------|--------|-------------| -| 1 | ceed0e5 | 941.252ms (+0.0%) | keep | baseline | -| 2 | 5269f43 | 1032.315ms (+9.7%) | discard | use stack buffer for token detokenization | -| 3 | 994e607 | 1004.435ms (+6.7%) | discard | slim turbo4b runtime profile | -| 4 | db92212 | 3027.145ms (+221.6%) | discard | switch to qwen-style chat template | -| 5 | caf4654 | 965.663ms (+2.6%) | discard | stop generation on default turn boundaries | -| 6 | e8402d2 | 925.587ms (-1.7%) | keep | disable llama perf counters in hot path | +| # | commit | template_score | status | description | +|---|--------|----------------|--------|-------------| +| 7 | 89ef220 | 20.0% (+0.0%) | keep | baseline | diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 79060bb..573ee49 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -5,3 +5,5 @@ {"run":4,"commit":"db92212","metric":3027.145,"metrics":{"p50_ms":974.937,"mean_ms":1467.026,"completion_tps_mean":41.360,"success_count":7},"status":"discard","description":"switch to qwen-style chat template","timestamp":1774558217,"segment":0} {"run":5,"commit":"caf4654","metric":965.663,"metrics":{"p50_ms":877.030,"mean_ms":897.251,"completion_tps_mean":53.686,"success_count":7},"status":"discard","description":"stop generation on default turn boundaries","timestamp":1774558444,"segment":0} {"run":6,"commit":"e8402d2","metric":925.587,"metrics":{"p50_ms":893.913,"mean_ms":899.895,"completion_tps_mean":53.618,"success_count":7},"status":"keep","description":"disable llama perf counters in hot path","timestamp":1774558642,"segment":0} +{"type":"config","name":"dynamic-chat-templates","metricName":"template_score","metricUnit":"%","bestDirection":"higher"} +{"run":7,"commit":"89ef220","metric":20.0,"metrics":{"passed_cases":1,"failed_cases":4,"total_cases":5},"status":"keep","description":"baseline","timestamp":1774559683,"segment":1} diff --git a/crates/inference-server/src/compat.rs b/crates/inference-server/src/compat.rs index a2cbff5..be94b8a 100644 --- a/crates/inference-server/src/compat.rs +++ b/crates/inference-server/src/compat.rs @@ -191,6 +191,15 @@ pub struct FinalizedAssistant { pub finish_reason: FinishReason, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ChatTemplate { + Generic, + Qwen, + Gemma, + Llama, + Kimi, +} + pub fn authorize_request( headers: &HeaderMap, config: &ServerConfig, @@ -294,6 +303,7 @@ pub fn validate_request( } pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> PreparedPrompt { + let template = select_chat_template(req, config); let mut prompt = String::with_capacity(1024); let has_system = req .messages @@ -301,52 +311,46 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre .any(|message| matches!(message.role, Role::System | Role::Developer)); if !has_system { - prompt.push_str("System: "); - prompt.push_str(&config.default_system_prompt); - prompt.push('\n'); + append_chat_message( + &mut prompt, + template, + Role::System, + None, + &config.default_system_prompt, + ); } if let Some(format) = &req.response_format { - append_response_format_instruction(&mut prompt, format); + append_response_format_instruction(&mut prompt, template, format); } if !req.tools.is_empty() { - append_tool_instructions(&mut prompt, &req.tools, req.tool_choice.as_ref()); + append_tool_instructions(&mut prompt, template, &req.tools, req.tool_choice.as_ref()); } for msg in &req.messages { - let role = match msg.role { - Role::System => "System", - Role::Developer => "Developer", - Role::User => "User", - Role::Assistant => "Assistant", - Role::Tool => "Tool", - }; - prompt.push_str(role); - if let Some(name) = &msg.name { - prompt.push_str("("); - prompt.push_str(name); - prompt.push(')'); - } - prompt.push_str(": "); - - if let Some(tool_calls) = &msg.tool_calls { - let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned()); - prompt.push_str(&tool_json); + let mut content = if let Some(tool_calls) = &msg.tool_calls { + serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned()) } else { - prompt.push_str(msg.text()); - } + msg.text().to_owned() + }; if let Some(tool_call_id) = &msg.tool_call_id { - prompt.push_str(" [tool_call_id="); - prompt.push_str(tool_call_id); - prompt.push(']'); + content.push_str(" [tool_call_id="); + content.push_str(tool_call_id); + content.push(']'); } - prompt.push('\n'); + append_chat_message( + &mut prompt, + template, + msg.role.clone(), + msg.name.as_deref(), + &content, + ); } - prompt.push_str("Assistant:"); + append_assistant_prefix(&mut prompt, template); PreparedPrompt { prompt, @@ -395,17 +399,47 @@ pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec { .collect() } -fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) { +fn select_chat_template(req: &ChatCompletionRequest, config: &ServerConfig) -> ChatTemplate { + let model_id = config + .resolve_model(&req.model) + .map(|model| model.root.as_str()) + .unwrap_or(req.model.as_str()); + let normalized = model_id.to_ascii_lowercase(); + + if normalized.contains("qwen") { + ChatTemplate::Qwen + } else if normalized.contains("gemma") { + ChatTemplate::Gemma + } else if normalized.contains("llama") { + ChatTemplate::Llama + } else if normalized.contains("kimi") { + ChatTemplate::Kimi + } else { + ChatTemplate::Generic + } +} + +fn append_response_format_instruction( + prompt: &mut String, + template: ChatTemplate, + format: &ResponseFormat, +) { match format.format_type.as_str() { "json_object" => { - prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n"); + append_chat_message( + prompt, + template, + Role::System, + None, + "Return valid JSON only. Do not wrap it in markdown fences.", + ); } "json_schema" => { - prompt.push_str("System: Return valid JSON only and follow this schema exactly: "); + let mut content = "Return valid JSON only and follow this schema exactly: ".to_owned(); if let Some(schema) = &format.json_schema { - prompt.push_str(&schema.to_string()); + content.push_str(&schema.to_string()); } - prompt.push('\n'); + append_chat_message(prompt, template, Role::System, None, &content); } _ => {} } @@ -413,6 +447,7 @@ fn append_response_format_instruction(prompt: &mut String, format: &ResponseForm fn append_tool_instructions( prompt: &mut String, + template: ChatTemplate, tools: &[ToolDefinition], tool_choice: Option<&ToolChoice>, ) { @@ -427,32 +462,150 @@ fn append_tool_instructions( }) .collect(); - prompt.push_str("System: Available tools: "); - prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())); - prompt.push('\n'); - prompt - .push_str("System: If you need a tool, respond with JSON only using one of these shapes: "); - prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#); - prompt.push_str(" or "); - prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#); - prompt.push_str(". If no tool is needed, answer normally.\n"); + let mut available_tools = "Available tools: ".to_owned(); + available_tools + .push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())); + append_chat_message(prompt, template, Role::System, None, &available_tools); + + let tool_usage = format!( + "If you need a tool, respond with JSON only using one of these shapes: {} or {}. If no tool is needed, answer normally.", + r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#, + r#"{"name":"tool_name","arguments":{}}"# + ); + append_chat_message(prompt, template, Role::System, None, &tool_usage); match tool_choice { Some(ToolChoice::Mode(mode)) if mode == "required" => { - prompt.push_str("System: You must call a tool before replying.\n"); + append_chat_message( + prompt, + template, + Role::System, + None, + "You must call a tool before replying.", + ); } Some(ToolChoice::Mode(mode)) if mode == "none" => { - prompt.push_str("System: Do not call tools in this response.\n"); + append_chat_message( + prompt, + template, + Role::System, + None, + "Do not call tools in this response.", + ); } Some(ToolChoice::Specific(choice)) => { - prompt.push_str("System: You must call the tool named "); - prompt.push_str(&choice.function.name); - prompt.push_str(".\n"); + let content = format!("You must call the tool named {}.", choice.function.name); + append_chat_message(prompt, template, Role::System, None, &content); } _ => {} } } +fn append_chat_message( + prompt: &mut String, + template: ChatTemplate, + role: Role, + name: Option<&str>, + content: &str, +) { + let content = if let Some(name) = name { + format!("{} ({})", content, name) + } else { + content.to_owned() + }; + + match template { + ChatTemplate::Generic => { + prompt.push_str(generic_role_name(&role)); + prompt.push_str(": "); + prompt.push_str(&content); + prompt.push('\n'); + } + ChatTemplate::Qwen => { + prompt.push_str("<|im_start|>"); + prompt.push_str(qwen_role_name(&role)); + prompt.push('\n'); + prompt.push_str(&content); + prompt.push_str("<|im_end|>\n"); + } + ChatTemplate::Gemma => { + prompt.push_str(""); + prompt.push_str(gemma_role_name(&role)); + prompt.push('\n'); + prompt.push_str(&content); + prompt.push_str("\n"); + } + ChatTemplate::Llama => match role { + Role::Assistant => { + prompt.push_str(&content); + prompt.push('\n'); + } + _ => { + prompt.push_str("[INST] "); + if matches!(role, Role::System | Role::Developer) { + prompt.push_str("<>\n"); + prompt.push_str(&content); + prompt.push_str("\n<>"); + } else { + prompt.push_str(&content); + } + prompt.push_str(" [/INST]\n"); + } + }, + ChatTemplate::Kimi => { + prompt.push_str(kimi_role_marker(&role)); + prompt.push('\n'); + prompt.push_str(&content); + prompt.push('\n'); + } + } +} + +fn append_assistant_prefix(prompt: &mut String, template: ChatTemplate) { + match template { + ChatTemplate::Generic => prompt.push_str("Assistant:"), + ChatTemplate::Qwen => prompt.push_str("<|im_start|>assistant\n"), + ChatTemplate::Gemma => prompt.push_str("model\n"), + ChatTemplate::Llama => prompt.push_str("[INST] "), + ChatTemplate::Kimi => prompt.push_str("<|im_assistant|>\n"), + } +} + +fn generic_role_name(role: &Role) -> &'static str { + match role { + Role::System => "System", + Role::Developer => "Developer", + Role::User => "User", + Role::Assistant => "Assistant", + Role::Tool => "Tool", + } +} + +fn qwen_role_name(role: &Role) -> &'static str { + match role { + Role::System | Role::Developer => "system", + Role::User => "user", + Role::Assistant => "assistant", + Role::Tool => "tool", + } +} + +fn gemma_role_name(role: &Role) -> &'static str { + match role { + Role::Assistant => "model", + Role::System | Role::Developer | Role::User | Role::Tool => "user", + } +} + +fn kimi_role_marker(role: &Role) -> &'static str { + match role { + Role::System | Role::Developer => "<|im_system|>", + Role::User => "<|im_user|>", + Role::Assistant => "<|im_assistant|>", + Role::Tool => "<|im_tool|>", + } +} + fn parse_tool_calls(raw_output: &str) -> Option> { let trimmed = raw_output.trim(); if trimmed.is_empty() { diff --git a/experiments/worklog.md b/experiments/worklog.md index 829f671..a91f397 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -61,3 +61,15 @@ - Result: p95_ms=3027.145, p50_ms=974.937, mean_ms=1467.026, completion_tps_mean=41.360, success_count=7. - Insight: The template did not make typical requests much faster and introduced severe tail latency, likely because some generations stopped much later or followed a worse decoding path for this server/model combination. - Next: revert this template change and focus on experiments that directly bound generation length or improve stop behavior without overhauling the prompt format. + +## Segment 1: Dynamic Chat Templates +- New objective: add dynamic prompt templates by model family with built-in coverage for Qwen, Gemma, Llama, Kimi, and a generic fallback. +- Benchmark driver: `./autoresearch.sh` now runs ignored template-specific tests and reports `template_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule for this segment is test-driven: higher `template_score` wins, while the regular non-ignored test suite must remain green. + +### Run 7: baseline - template_score=20.0 (KEEP) +- Timestamp: 2026-03-26 18:14 +- What changed: Re-initialized autoresearch to a new segment focused on dynamic chat templates, added a dedicated ignored test suite for Qwen, Gemma, Llama, Kimi, and generic fallback behavior, and switched the benchmark harness to score that suite. +- Result: template_score=20.0, passed_cases=1, failed_cases=4, total_cases=5. +- Insight: Only the generic fallback works today; all named family-template expectations fail because prompt rendering is still generic and model-agnostic. +- Next: add a minimal model-family detector and template registry in `crates/inference-server/src/compat.rs`, then route `prepare_prompt()` through it.