chore: initialize template autoresearch segment

experiments/bench_chat_templates.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+TEST_FILTER = "dynamic_template_"
+TEST_PATTERN = re.compile(
+    r"^test\s+(.+dynamic_template_[^\s]+)\s+\.\.\.\s+(ok|FAILED)$"
+)
+
+
+def run_suite(root: Path) -> tuple[int, int, int]:
+    command = [
+        "cargo",
+        "test",
+        "-p",
+        "inference-server",
+        TEST_FILTER,
+        "--",
+        "--ignored",
+        "--nocapture",
+    ]
+    completed = subprocess.run(
+        command,
+        cwd=root,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+
+    output = completed.stdout + completed.stderr
+    print(output, end="")
+
+    seen: dict[str, str] = {}
+    for line in output.splitlines():
+        match = TEST_PATTERN.match(line.strip())
+        if match:
+            seen[match.group(1)] = match.group(2)
+
+    if not seen and completed.returncode != 0:
+        raise SystemExit(completed.returncode)
+
+    total = len(seen)
+    passed = sum(1 for status in seen.values() if status == "ok")
+    failed = total - passed
+    score = round((passed / total) * 100.0, 3) if total else 0.0
+
+    print(f"METRIC template_score={score}")
+    print(f"METRIC passed_cases={passed}")
+    print(f"METRIC failed_cases={failed}")
+    print(f"METRIC total_cases={total}")
+    return passed, failed, total
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", required=True)
+    args = parser.parse_args()
+
+    root = Path(args.root)
+    passed, failed, total = run_suite(root)
+    return 0 if total > 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())