From 9555a029b97d68ae825b69c9c3a311adeb7cbdbd Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 18:53:11 -0300 Subject: [PATCH] chore: initialize cli autoresearch segment --- Cargo.toml | 2 + autoresearch.md | 56 +++++----- autoresearch.sh | 4 +- crates/localllm-cli/Cargo.toml | 14 +++ crates/localllm-cli/src/main.rs | 160 ++++++++++++++++++++++++++++ experiments/bench_chat_templates.py | 70 ------------ experiments/bench_cli_surface.py | 72 +++++++++++++ experiments/worklog.md | 6 ++ 8 files changed, 280 insertions(+), 104 deletions(-) create mode 100644 crates/localllm-cli/Cargo.toml create mode 100644 crates/localllm-cli/src/main.rs delete mode 100644 experiments/bench_chat_templates.py create mode 100644 experiments/bench_cli_surface.py diff --git a/Cargo.toml b/Cargo.toml index 39d1c77..3c37bdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/inference-server", "crates/api-contracts", "crates/metrics", + "crates/localllm-cli", ] [workspace.package] @@ -44,6 +45,7 @@ chrono = { version = "0.4", features = ["serde"] } futures = "0.3" tokio-stream = "0.1" bytes = "1" +clap = { version = "4", features = ["derive"] } [profile.dev] opt-level = 0 diff --git a/autoresearch.md b/autoresearch.md index a2c1285..dc5ca7e 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -1,52 +1,44 @@ -# Autoresearch: dynamic-chat-templates +# Autoresearch: localllm-cli-control-surface ## Objective -Implement a dynamic chat-template system that selects prompt rendering by model family -for local OpenAI-compatible `POST /v1/chat/completions` requests. The server should ship -useful built-in templates for common open-source families such as Qwen, Gemma, Llama, -and Kimi, while preserving a safe generic fallback for unknown models. +Implement a full management CLI on top of the current local backend version, starting with +a professional command surface for status, model operations, auth, config, logs, health, +and throughput benchmarking. This segment focuses first on the CLI surface and contracts, +so later experiments can add real execution behavior without redesigning the UX shape. ## Metrics -- **Primary**: template_score (percent, higher is better) +- **Primary**: cli_surface_score (percent, higher is better) - **Secondary**: passed_cases, failed_cases, total_cases ## How to Run -`./autoresearch.sh` - runs targeted ignored tests for dynamic chat-template behavior and outputs `METRIC name=number` lines. +`./autoresearch.sh` - runs targeted ignored tests for CLI command-surface behavior and outputs `METRIC name=number` lines. ## Files in Scope -- `crates/inference-server/src/compat.rs` - prompt preparation, model catalog, template selection, family detection, and template-focused tests -- `crates/inference-server/src/routes.rs` - route-level integration if template selection needs model-aware wiring -- `crates/inference-server/src/main.rs` - server boot wiring if template config/env support is needed -- `crates/api-contracts/src/chat.rs` - request/response roles and content handling that templates consume -- `experiments/bench_chat_templates.py` - benchmark harness for the autoresearch loop +- `crates/localllm-cli/` - new CLI crate, parser, command UX, output modes, and CLI-focused tests +- `crates/inference-server/src/main.rs` - current backend env/config seam the CLI will eventually manage +- `crates/inference-server/src/routes.rs` - existing inference endpoints the CLI can interrogate today +- `crates/inference-server/src/compat.rs` - existing auth/config knobs the CLI must understand and expose coherently +- `crates/api-contracts/` - shared contracts the CLI may eventually consume +- `experiments/bench_cli_surface.py` - benchmark harness for the autoresearch loop - `autoresearch.sh` - loop benchmark entrypoint and fast preflight ## Off Limits - `models/` model binaries - `vendor/` third-party sources and prebuilt llama binaries -- `registry/` Python registry service +- `registry/` Python registry service unless a later CLI experiment explicitly needs to replace or wire it - `ui/` frontend files -- unrelated latency tuning or runtime/engine experiments unless they are required to support template selection correctly +- Tauri app implementation in this segment +- unrelated latency or template experiments ## Constraints -- No new external dependencies. -- Preserve OpenAI-compatible behavior already implemented. +- Preserve the currently working backend behavior. - Regular test suite must stay green for keeps. -- Prefer explicit, maintainable template definitions over opaque heuristics. -- Unknown models must continue to work through a generic fallback template. -- The benchmark is test-driven: keep/discard follows the dynamic-template suite score first. +- Prefer a stable, professional CLI contract over premature execution internals. +- Commands should map cleanly to the product plan discussed with the user: status, models, auth, config, bench, logs, doctor. +- The benchmark is test-driven: keep/discard follows the CLI surface suite score first. +- A justified CLI-focused dependency is acceptable if it reduces maintenance cost. ## What's Been Tried -- Segment 1 switches the autoresearch target from latency to dynamic chat-template selection by model family. -- The current server still renders every prompt through one generic `System:/User:/Assistant:` formatter in `crates/inference-server/src/compat.rs`. -- `prepare_prompt()` is the cleanest insertion point for dynamic templates because both blocking and streaming paths already consume its rendered string. -- `ModelCatalog` currently resolves aliases only; it has no model-family metadata or template registry yet. -- The benchmark for this segment is a targeted ignored test suite that scores family detection/template rendering coverage for Qwen, Gemma, Llama, Kimi, and the generic fallback. -- Baseline run 7 scored only `20.0%` because just the generic fallback path passed; Qwen, Gemma, Llama, and Kimi were still rendered through the old generic formatter. -- Run 8 (`ceb83f2`) added family detection plus built-in template selection inside `compat.rs` and immediately raised coverage to `100.0%` (5/5 passing), making it the current best for segment 1. -- Segment 2 raises the bar by checking stronger Llama structure, especially whether system+user are merged into the first instruction block and whether the prompt avoids leaving an open `[INST]` prefix at generation time. -- Run 10 (`fd6d35e`) hardened the Llama renderer with a dedicated formatting path that folds pending system/developer context into the next instruction block and eliminates the dangling open `[INST]` suffix, raising segment 2 from `71.429%` to `100.0%` (7/7 passing). -- Segment 3 raises the bar again around alias-driven family detection, specifically ensuring a family hint carried only by an alias still selects the right template even when the canonical/root model id is generic. -- Run 12 (`140c9b2`) fixed alias-driven family detection by checking the requested model string before falling back to resolved alias/root metadata, raising segment 3 from `88.889%` to `100.0%` (9/9 passing). -- Segment 4 raises the bar around family-name variants, starting with Moonshot-branded Kimi aliases so common `moonshot` names select Kimi markers while unrelated lookalikes still fall back safely to `Generic`. -- Run 14 (`f200733`) extended family detection so `moonshot` variant names map to the Kimi template, raising segment 4 from `90.909%` to `100.0%` (11/11 passing) without weakening generic fallback behavior. +- Segment 5 pivots autoresearch from template hardening to productization, specifically a complete CLI-first control surface on top of the current backend version. +- The current repo has no CLI crate, no Tauri app, and no control-plane crate; the backend is still configured almost entirely by `LOCALLLM_*` environment variables and exposes only `/health`, `/v1/models`, and `/v1/chat/completions`. +- The first benchmark for this segment is intentionally surface-oriented: ignored tests score whether the CLI presents the expected professional command set before deeper execution behavior is implemented. diff --git a/autoresearch.sh b/autoresearch.sh index 7afb276..8022c71 100644 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -6,7 +6,7 @@ ROOT="$SCRIPT_DIR" mkdir -p "$ROOT/experiments" echo "[INFO] Fast preflight (cargo check)..." >&2 -cargo check -p inference-server --quiet +cargo check -p localllm-cli --quiet -python "$ROOT/experiments/bench_chat_templates.py" \ +python "$ROOT/experiments/bench_cli_surface.py" \ --root "$ROOT" diff --git a/crates/localllm-cli/Cargo.toml b/crates/localllm-cli/Cargo.toml new file mode 100644 index 0000000..c124e9b --- /dev/null +++ b/crates/localllm-cli/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "localllm-cli" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true + +[[bin]] +name = "localllm" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs new file mode 100644 index 0000000..93c87ae --- /dev/null +++ b/crates/localllm-cli/src/main.rs @@ -0,0 +1,160 @@ +use clap::{Args, Parser, Subcommand, ValueEnum}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum OutputFormat { + Text, + Json, +} + +#[derive(Debug, Parser)] +#[command(name = "localllm", about = "Manage the local LLM backend")] +struct Cli { + #[arg(long, global = true, default_value = "http://127.0.0.1:8080")] + server: String, + + #[arg(long, global = true)] + api_key: Option, + + #[arg(long, global = true, value_enum, default_value_t = OutputFormat::Text)] + format: OutputFormat, + + #[command(subcommand)] + command: Commands, +} + +#[derive(Debug, Subcommand)] +enum Commands { + Status, + Models(ModelsCommand), +} + +#[derive(Debug, Args)] +struct ModelsCommand { + #[command(subcommand)] + command: ModelsSubcommand, +} + +#[derive(Debug, Subcommand)] +enum ModelsSubcommand { + List, +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + println!("CLI scaffold ready for {:?} at {}", cli.command, cli.server); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::Cli; + use clap::Parser; + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_command_supports_global_output_flags() { + let parsed = Cli::try_parse_from(["localllm", "--format", "json", "status"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_list_command_is_available() { + let parsed = Cli::try_parse_from(["localllm", "models", "list"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_scan_command_is_available() { + let parsed = Cli::try_parse_from(["localllm", "models", "scan"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_load_command_accepts_model_and_alias() { + let parsed = Cli::try_parse_from([ + "localllm", + "models", + "load", + "./models/qwen.gguf", + "--alias", + "qwen-coder", + ]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_watch_command_is_available() { + let parsed = Cli::try_parse_from(["localllm", "models", "watch"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_download_command_accepts_hf_repo() { + let parsed = Cli::try_parse_from([ + "localllm", + "models", + "download", + "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", + ]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_auth_enable_command_supports_mode_and_key() { + let parsed = Cli::try_parse_from([ + "localllm", + "auth", + "enable", + "--mode", + "protected", + "--api-key", + "secret", + ]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_config_set_command_supports_key_value_updates() { + let parsed = + Cli::try_parse_from(["localllm", "config", "set", "default_model", "qwen-coder"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_throughput_command_accepts_sample_controls() { + let parsed = Cli::try_parse_from([ + "localllm", + "bench", + "throughput", + "--samples", + "5", + "--prompt-tokens", + "256", + "--completion-tokens", + "128", + ]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_logs_command_is_available() { + let parsed = Cli::try_parse_from(["localllm", "logs"]); + assert!(parsed.is_ok()); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_doctor_command_is_available() { + let parsed = Cli::try_parse_from(["localllm", "doctor"]); + assert!(parsed.is_ok()); + } +} diff --git a/experiments/bench_chat_templates.py b/experiments/bench_chat_templates.py deleted file mode 100644 index 86b61fc..0000000 --- a/experiments/bench_chat_templates.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import re -import subprocess -import sys -from pathlib import Path - - -TEST_FILTER = "dynamic_template_" -TEST_PATTERN = re.compile( - r"^test\s+(.+dynamic_template_[^\s]+)\s+\.\.\.\s+(ok|FAILED)$" -) - - -def run_suite(root: Path) -> tuple[int, int, int]: - command = [ - "cargo", - "test", - "-p", - "inference-server", - TEST_FILTER, - "--", - "--ignored", - "--nocapture", - ] - completed = subprocess.run( - command, - cwd=root, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - ) - - output = completed.stdout + completed.stderr - print(output, end="") - - seen: dict[str, str] = {} - for line in output.splitlines(): - match = TEST_PATTERN.match(line.strip()) - if match: - seen[match.group(1)] = match.group(2) - - if not seen and completed.returncode != 0: - raise SystemExit(completed.returncode) - - total = len(seen) - passed = sum(1 for status in seen.values() if status == "ok") - failed = total - passed - score = round((passed / total) * 100.0, 3) if total else 0.0 - - print(f"METRIC template_score={score}") - print(f"METRIC passed_cases={passed}") - print(f"METRIC failed_cases={failed}") - print(f"METRIC total_cases={total}") - return passed, failed, total - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--root", required=True) - args = parser.parse_args() - - root = Path(args.root) - passed, failed, total = run_suite(root) - return 0 if total > 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/bench_cli_surface.py b/experiments/bench_cli_surface.py new file mode 100644 index 0000000..c7b6834 --- /dev/null +++ b/experiments/bench_cli_surface.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import argparse +import re +import subprocess +import sys +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + args = parser.parse_args() + + root = Path(args.root) + command = [ + "cargo", + "test", + "-p", + "localllm-cli", + "cli_surface_", + "--", + "--ignored", + "--nocapture", + ] + + completed = subprocess.run( + command, + cwd=root, + text=True, + capture_output=True, + ) + + output = (completed.stdout or "") + (completed.stderr or "") + sys.stdout.write(output) + + passed_match = re.search( + r"test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;", output + ) + failed_match = re.search( + r"test result: FAILED\. (\d+) passed; (\d+) failed; (\d+) ignored;", output + ) + + passed_cases = 0 + failed_cases = 0 + total_cases = 0 + + if passed_match: + passed_cases = int(passed_match.group(1)) + failed_cases = int(passed_match.group(2)) + total_cases = passed_cases + failed_cases + elif failed_match: + passed_cases = int(failed_match.group(1)) + failed_cases = int(failed_match.group(2)) + total_cases = passed_cases + failed_cases + + if completed.returncode != 0 and total_cases == 0: + print("METRIC cli_surface_score=0.0") + print("METRIC passed_cases=0") + print("METRIC failed_cases=0") + print("METRIC total_cases=0") + return completed.returncode + + score = 0.0 if total_cases == 0 else round((passed_cases / total_cases) * 100.0, 3) + print(f"METRIC cli_surface_score={score}") + print(f"METRIC passed_cases={passed_cases}") + print(f"METRIC failed_cases={failed_cases}") + print(f"METRIC total_cases={total_cases}") + return 0 if total_cases > 0 else completed.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/worklog.md b/experiments/worklog.md index 0a093ef..3e57099 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -138,3 +138,9 @@ - Result: template_score=100.0, passed_cases=11, failed_cases=0, total_cases=11. - Insight: A narrow Moonshot variant match closes the Kimi-family naming gap cleanly without breaking safe generic fallback for unrelated names such as `moonshine-local`. - Next: raise the benchmark around multi-turn continuity, tool-role rendering, or family-specific response-format instructions. + +## Segment 5: CLI Control Surface +- New objective: build a complete management CLI first, ahead of the Tauri app, on top of the current backend version. +- Benchmark driver: `./autoresearch.sh` now runs ignored CLI-surface tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule is test-driven: higher `cli_surface_score` wins while the regular non-ignored suite remains green. +- This segment starts by benchmarking the command surface itself before deeper execution and backend-admin semantics are implemented.