chore: initialize cli autoresearch segment

experiments/bench_chat_templates.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import re
-import subprocess
-import sys
-from pathlib import Path
-
-
-TEST_FILTER = "dynamic_template_"
-TEST_PATTERN = re.compile(
-    r"^test\s+(.+dynamic_template_[^\s]+)\s+\.\.\.\s+(ok|FAILED)$"
-)
-
-
-def run_suite(root: Path) -> tuple[int, int, int]:
-    command = [
-        "cargo",
-        "test",
-        "-p",
-        "inference-server",
-        TEST_FILTER,
-        "--",
-        "--ignored",
-        "--nocapture",
-    ]
-    completed = subprocess.run(
-        command,
-        cwd=root,
-        capture_output=True,
-        text=True,
-        encoding="utf-8",
-        errors="replace",
-    )
-
-    output = completed.stdout + completed.stderr
-    print(output, end="")
-
-    seen: dict[str, str] = {}
-    for line in output.splitlines():
-        match = TEST_PATTERN.match(line.strip())
-        if match:
-            seen[match.group(1)] = match.group(2)
-
-    if not seen and completed.returncode != 0:
-        raise SystemExit(completed.returncode)
-
-    total = len(seen)
-    passed = sum(1 for status in seen.values() if status == "ok")
-    failed = total - passed
-    score = round((passed / total) * 100.0, 3) if total else 0.0
-
-    print(f"METRIC template_score={score}")
-    print(f"METRIC passed_cases={passed}")
-    print(f"METRIC failed_cases={failed}")
-    print(f"METRIC total_cases={total}")
-    return passed, failed, total
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--root", required=True)
-    args = parser.parse_args()
-
-    root = Path(args.root)
-    passed, failed, total = run_suite(root)
-    return 0 if total > 0 else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

experiments/bench_cli_surface.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", required=True)
+    args = parser.parse_args()
+
+    root = Path(args.root)
+    command = [
+        "cargo",
+        "test",
+        "-p",
+        "localllm-cli",
+        "cli_surface_",
+        "--",
+        "--ignored",
+        "--nocapture",
+    ]
+
+    completed = subprocess.run(
+        command,
+        cwd=root,
+        text=True,
+        capture_output=True,
+    )
+
+    output = (completed.stdout or "") + (completed.stderr or "")
+    sys.stdout.write(output)
+
+    passed_match = re.search(
+        r"test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
+    )
+    failed_match = re.search(
+        r"test result: FAILED\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
+    )
+
+    passed_cases = 0
+    failed_cases = 0
+    total_cases = 0
+
+    if passed_match:
+        passed_cases = int(passed_match.group(1))
+        failed_cases = int(passed_match.group(2))
+        total_cases = passed_cases + failed_cases
+    elif failed_match:
+        passed_cases = int(failed_match.group(1))
+        failed_cases = int(failed_match.group(2))
+        total_cases = passed_cases + failed_cases
+
+    if completed.returncode != 0 and total_cases == 0:
+        print("METRIC cli_surface_score=0.0")
+        print("METRIC passed_cases=0")
+        print("METRIC failed_cases=0")
+        print("METRIC total_cases=0")
+        return completed.returncode
+
+    score = 0.0 if total_cases == 0 else round((passed_cases / total_cases) * 100.0, 3)
+    print(f"METRIC cli_surface_score={score}")
+    print(f"METRIC passed_cases={passed_cases}")
+    print(f"METRIC failed_cases={failed_cases}")
+    print(f"METRIC total_cases={total_cases}")
+    return 0 if total_cases > 0 else completed.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

experiments/worklog.md

@@ -138,3 +138,9 @@
 - Result: template_score=100.0, passed_cases=11, failed_cases=0, total_cases=11.
 - Insight: A narrow Moonshot variant match closes the Kimi-family naming gap cleanly without breaking safe generic fallback for unrelated names such as `moonshine-local`.
 - Next: raise the benchmark around multi-turn continuity, tool-role rendering, or family-specific response-format instructions.
+
+## Segment 5: CLI Control Surface
+- New objective: build a complete management CLI first, ahead of the Tauri app, on top of the current backend version.
+- Benchmark driver: `./autoresearch.sh` now runs ignored CLI-surface tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule is test-driven: higher `cli_surface_score` wins while the regular non-ignored suite remains green.
+- This segment starts by benchmarking the command surface itself before deeper execution and backend-admin semantics are implemented.