From 9bf79ff68c393e4d5db159c765208213e3e1b340 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 18:26:40 -0300 Subject: [PATCH] chore: expand alias template benchmark coverage --- autoresearch-dashboard.md | 10 +++--- autoresearch.jsonl | 3 ++ autoresearch.md | 2 ++ crates/inference-server/src/compat.rs | 47 +++++++++++++++++++++++++++ experiments/worklog.md | 19 +++++++++++ 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/autoresearch-dashboard.md b/autoresearch-dashboard.md index 3605502..bc93cb6 100644 --- a/autoresearch-dashboard.md +++ b/autoresearch-dashboard.md @@ -1,10 +1,10 @@ -# Autoresearch Dashboard: dynamic-chat-templates +# Autoresearch Dashboard: dynamic-chat-templates-hardening **Runs:** 2 | **Kept:** 2 | **Discarded:** 0 | **Crashed:** 0 -**Baseline:** template_score: 20.0% (#7) -**Best:** template_score: 100.0% (#8, +400.0%) +**Baseline:** template_score: 71.429% (#9) +**Best:** template_score: 100.0% (#10, +40.0%) | # | commit | template_score | status | description | |---|--------|----------------|--------|-------------| -| 7 | 89ef220 | 20.0% (+0.0%) | keep | baseline | -| 8 | ceb83f2 | 100.0% (+400.0%) | keep | add family-based chat templates | +| 9 | 5f3ed5e | 71.429% (+0.0%) | keep | baseline | +| 10 | fd6d35e | 100.0% (+40.0%) | keep | harden llama chat template rendering | diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 9c0e0bc..06a95e1 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -8,3 +8,6 @@ {"type":"config","name":"dynamic-chat-templates","metricName":"template_score","metricUnit":"%","bestDirection":"higher"} {"run":7,"commit":"89ef220","metric":20.0,"metrics":{"passed_cases":1,"failed_cases":4,"total_cases":5},"status":"keep","description":"baseline","timestamp":1774559683,"segment":1} {"run":8,"commit":"ceb83f2","metric":100.0,"metrics":{"passed_cases":5,"failed_cases":0,"total_cases":5},"status":"keep","description":"add family-based chat templates","timestamp":1774559814,"segment":1} +{"type":"config","name":"dynamic-chat-templates-hardening","metricName":"template_score","metricUnit":"%","bestDirection":"higher"} +{"run":9,"commit":"5f3ed5e","metric":71.429,"metrics":{"passed_cases":5,"failed_cases":2,"total_cases":7},"status":"keep","description":"baseline","timestamp":1774560085,"segment":2} +{"run":10,"commit":"fd6d35e","metric":100.0,"metrics":{"passed_cases":7,"failed_cases":0,"total_cases":7},"status":"keep","description":"harden llama chat template rendering","timestamp":1774560229,"segment":2} diff --git a/autoresearch.md b/autoresearch.md index a84160d..4c7618e 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -44,3 +44,5 @@ and Kimi, while preserving a safe generic fallback for unknown models. - The benchmark for this segment is a targeted ignored test suite that scores family detection/template rendering coverage for Qwen, Gemma, Llama, Kimi, and the generic fallback. - Baseline run 7 scored only `20.0%` because just the generic fallback path passed; Qwen, Gemma, Llama, and Kimi were still rendered through the old generic formatter. - Run 8 (`ceb83f2`) added family detection plus built-in template selection inside `compat.rs` and immediately raised coverage to `100.0%` (5/5 passing), making it the current best for segment 1. +- Segment 2 raises the bar by checking stronger Llama structure, especially whether system+user are merged into the first instruction block and whether the prompt avoids leaving an open `[INST]` prefix at generation time. +- Run 10 (`fd6d35e`) hardened the Llama renderer with a dedicated formatting path that folds pending system/developer context into the next instruction block and eliminates the dangling open `[INST]` suffix, raising segment 2 from `71.429%` to `100.0%` (7/7 passing). diff --git a/crates/inference-server/src/compat.rs b/crates/inference-server/src/compat.rs index b5201ca..09896f6 100644 --- a/crates/inference-server/src/compat.rs +++ b/crates/inference-server/src/compat.rs @@ -1068,4 +1068,51 @@ mod tests { assert!(prepared.prompt.contains("User: hello")); assert!(prepared.prompt.ends_with("Assistant:")); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn dynamic_template_alias_family_can_override_generic_canonical_id() { + let config = ServerConfig { + model_catalog: ModelCatalog::new( + "my-local-model".to_owned(), + vec!["qwen-coder".to_owned()], + ), + ..base_config() + }; + let request = parse_request( + r#"{ + "model": "qwen-coder", + "messages": [{"role":"user","content":"hello"}] + }"#, + ); + + let prepared = prepare_prompt(&request, &config); + assert!(prepared.prompt.contains("<|im_start|>system\n")); + assert!(prepared.prompt.ends_with("<|im_start|>assistant\n")); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn dynamic_template_generic_alias_stays_generic_without_family_hint() { + let config = ServerConfig { + model_catalog: ModelCatalog::new( + "my-local-model".to_owned(), + vec!["assistant-prod".to_owned()], + ), + ..base_config() + }; + let request = parse_request( + r#"{ + "model": "assistant-prod", + "messages": [{"role":"user","content":"hello"}] + }"#, + ); + + let prepared = prepare_prompt(&request, &config); + assert!(prepared + .prompt + .contains("System: Default coding system prompt")); + assert!(prepared.prompt.contains("User: hello")); + assert!(prepared.prompt.ends_with("Assistant:")); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index af8a7fa..a71106d 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -81,3 +81,22 @@ - Result: template_score=100.0, passed_cases=5, failed_cases=0, total_cases=5. - Insight: The current seam in `prepare_prompt()` is sufficient for broad family-based template support; no runtime or routing changes were needed for the first successful implementation. - Next: harden the design by adding alias-driven family detection, safer fallback behavior for mixed-case/variant model names, and more coverage for tool-call and response-format rendering under non-generic templates. + +## Segment 2: Dynamic Chat Template Hardening +- New objective: keep family-based template selection, but harden structure-sensitive formatting for instruct-style models, starting with Llama. +- Benchmark driver: `./autoresearch.sh` still runs the ignored template suite, now with stricter Llama assertions. +- Keep/discard rule remains test-driven: higher `template_score` wins while the regular non-ignored suite stays green. + +### Run 9: baseline - template_score=71.429 (KEEP) +- Timestamp: 2026-03-26 18:21 +- What changed: Expanded the ignored benchmark suite with two stronger Llama assertions that require system+user to be merged into the first instruction block and forbid leaving an open `[INST]` prefix at generation time. +- Result: template_score=71.429, passed_cases=5, failed_cases=2, total_cases=7. +- Insight: The current family detector is good enough, but the Llama renderer is still structurally naive; it over-produces instruction blocks and leaves the prompt in an invalid generation state. +- Next: refactor Llama rendering into a dedicated formatter that folds system context into the first user turn and emits a proper assistant generation prefix. + +### Run 10: harden llama chat template rendering - template_score=100.0 (KEEP) +- Timestamp: 2026-03-26 18:23 +- What changed: Reworked the Llama path in `crates/inference-server/src/compat.rs` into a dedicated renderer that accumulates pending system/developer content, folds it into the next user/tool instruction block, and stops emitting a dangling open assistant `[INST]` prefix. +- Result: template_score=100.0, passed_cases=7, failed_cases=0, total_cases=7. +- Insight: Llama needed a specialized formatter instead of the generic per-message helper; once the prompt was rendered as proper closed instruction blocks with folded system context, the hardening suite passed cleanly. +- Next: raise the benchmark again around multi-turn continuity, tool-role placement, or alias-driven family detection so the next experiments target real edge cases instead of already-solved happy paths.