From a3f55857005de8605cd97dd80039306b55e7e38c Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 18:59:01 -0300 Subject: [PATCH] chore: expand cli semantic benchmark coverage --- autoresearch.jsonl | 4 ++ autoresearch.md | 3 + crates/localllm-cli/src/main.rs | 100 ++++++++++++++++++++++++++++++-- experiments/worklog.md | 19 ++++++ 4 files changed, 121 insertions(+), 5 deletions(-) diff --git a/autoresearch.jsonl b/autoresearch.jsonl index c78b380..09386d0 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -17,3 +17,7 @@ {"type":"config","name":"dynamic-chat-templates-variants","metricName":"template_score","metricUnit":"%","bestDirection":"higher"} {"run":13,"commit":"ffab766","metric":90.909,"metrics":{"passed_cases":10,"failed_cases":1,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774560738,"segment":4} {"run":14,"commit":"f200733","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"detect moonshot as kimi family","timestamp":1774560829,"segment":4} +{"type":"config","name":"localllm-cli-control-surface","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":15,"commit":"fd773a9","metric":18.182,"metrics":{"passed_cases":2,"failed_cases":9,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774562027,"segment":5} +{"run":16,"commit":"70cc59a","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"expand cli command surface","timestamp":1774562138,"segment":5} +{"type":"config","name":"localllm-cli-semantic-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index dc5ca7e..60e884e 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -42,3 +42,6 @@ so later experiments can add real execution behavior without redesigning the UX - Segment 5 pivots autoresearch from template hardening to productization, specifically a complete CLI-first control surface on top of the current backend version. - The current repo has no CLI crate, no Tauri app, and no control-plane crate; the backend is still configured almost entirely by `LOCALLLM_*` environment variables and exposes only `/health`, `/v1/models`, and `/v1/chat/completions`. - The first benchmark for this segment is intentionally surface-oriented: ignored tests score whether the CLI presents the expected professional command set before deeper execution behavior is implemented. +- Baseline run 15 (`fd773a9`) scored only `18.182%`: the new CLI scaffold currently exposes `status` and `models list`, but the broader management surface (`models scan/load/watch/download`, `auth`, `config`, `bench`, `logs`, `doctor`) is still absent. +- Run 16 (`70cc59a`) expanded the Clap parser to the full intended command surface and raised segment 5 from `18.182%` to `100.0%` (11/11 passing), giving the CLI a stable UX contract before execution handlers are wired. +- Segment 6 raises the bar from command presence to semantic usefulness: the benchmark now checks whether key workflows produce meaningful command summaries that preserve defaults and high-value arguments, as a stepping stone toward real execution planning. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index 79f1185..1a9e5b9 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -180,12 +180,16 @@ struct LogsArgs { fn main() -> anyhow::Result<()> { let cli = Cli::parse(); - println!( - "CLI scaffold ready for {} at {}", + println!("CLI scaffold ready for {}", render_command_summary(&cli)); + Ok(()) +} + +fn render_command_summary(cli: &Cli) -> String { + format!( + "command={} server={}", command_summary(&cli.command), cli.server - ); - Ok(()) + ) } fn command_summary(command: &Commands) -> &'static str { @@ -202,7 +206,7 @@ fn command_summary(command: &Commands) -> &'static str { #[cfg(test)] mod tests { - use super::Cli; + use super::{render_command_summary, AuthMode, Cli, Commands}; use clap::Parser; #[test] @@ -312,4 +316,90 @@ mod tests { let parsed = Cli::try_parse_from(["localllm", "doctor"]); assert!(parsed.is_ok()); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_summary_includes_output_contract() { + let cli = Cli::try_parse_from(["localllm", "--format", "json", "status"]).unwrap(); + let summary = render_command_summary(&cli); + + assert!(summary.contains("status")); + assert!(summary.contains("format=json")); + assert!(summary.contains("models_dir=./models")); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_load_summary_includes_model_and_alias() { + let cli = Cli::try_parse_from([ + "localllm", + "models", + "load", + "./models/qwen.gguf", + "--alias", + "qwen-coder", + ]) + .unwrap(); + let summary = render_command_summary(&cli); + + assert!(summary.contains("models.load")); + assert!(summary.contains("./models/qwen.gguf")); + assert!(summary.contains("alias=qwen-coder")); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_auth_enable_summary_includes_mode_and_key_presence() { + let cli = Cli::try_parse_from([ + "localllm", + "auth", + "enable", + "--mode", + "local-only", + "--api-key", + "secret", + ]) + .unwrap(); + let summary = render_command_summary(&cli); + + assert!(summary.contains("auth.enable")); + assert!(summary.contains("mode=local-only")); + assert!(summary.contains("has_api_key=true")); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_throughput_summary_includes_default_budget() { + let cli = Cli::try_parse_from(["localllm", "bench", "throughput"]).unwrap(); + let summary = render_command_summary(&cli); + + assert!(summary.contains("bench.throughput")); + assert!(summary.contains("samples=3")); + assert!(summary.contains("prompt_tokens=256")); + assert!(summary.contains("completion_tokens=128")); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_logs_summary_includes_default_line_limit() { + let cli = Cli::try_parse_from(["localllm", "logs"]).unwrap(); + let summary = render_command_summary(&cli); + + assert!(summary.contains("logs")); + assert!(summary.contains("lines=200")); + } + + #[test] + fn auth_mode_parser_accepts_local_only() { + let cli = + Cli::try_parse_from(["localllm", "auth", "enable", "--mode", "local-only"]).unwrap(); + + match cli.command { + Commands::Auth(auth) => match auth.command { + super::AuthSubcommand::Enable(args) => assert_eq!(args.mode, AuthMode::LocalOnly), + other => panic!("unexpected auth command: {:?}", other), + }, + other => panic!("unexpected command: {:?}", other), + } + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index 3e57099..572e00d 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -144,3 +144,22 @@ - Benchmark driver: `./autoresearch.sh` now runs ignored CLI-surface tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule is test-driven: higher `cli_surface_score` wins while the regular non-ignored suite remains green. - This segment starts by benchmarking the command surface itself before deeper execution and backend-admin semantics are implemented. + +### Run 15: baseline - cli_surface_score=18.182 (KEEP) +- Timestamp: 2026-03-26 18:53 +- What changed: Initialized a new `crates/localllm-cli` crate with a minimal Clap-based CLI scaffold, switched the autoresearch harness from template tests to CLI surface tests, and defined the first ignored benchmark suite for professional command coverage. +- Result: cli_surface_score=18.182, passed_cases=2, failed_cases=9, total_cases=11. +- Insight: The scaffold already covers global flags plus `status` and `models list`, but almost the entire promised management surface is still missing. +- Next: add the remaining top-level command groups and model/auth/config/bench/logs/doctor subcommands so the CLI surface matches the intended product contract before wiring execution internals. + +### Run 16: expand cli command surface - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 18:55 +- What changed: Expanded `crates/localllm-cli/src/main.rs` into a broad Clap parser with top-level `status`, `models`, `auth`, `config`, `bench`, `logs`, and `doctor` commands plus the expected model/auth/config/throughput subcommands and global flags. +- Result: cli_surface_score=100.0, passed_cases=11, failed_cases=0, total_cases=11. +- Insight: The professional CLI contract can be locked in quickly at the parser layer before execution internals exist; Clap gives a maintainable foundation for the full management surface. +- Next: raise the benchmark from command presence to semantic correctness, starting with command-specific arguments, defaults, and output-contract expectations for the highest-value workflows. + +## Segment 6: CLI Semantic Contracts +- New objective: keep the CLI surface stable, but raise the benchmark so key commands preserve useful semantic detail such as defaults, high-value arguments, and summary/intention contracts. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.