chore: expand cli semantic benchmark coverage

autoresearch.jsonl

@@ -17,3 +17,7 @@
 {"type":"config","name":"dynamic-chat-templates-variants","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
 {"run":13,"commit":"ffab766","metric":90.909,"metrics":{"passed_cases":10,"failed_cases":1,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774560738,"segment":4}
 {"run":14,"commit":"f200733","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"detect moonshot as kimi family","timestamp":1774560829,"segment":4}
+{"type":"config","name":"localllm-cli-control-surface","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
+{"run":15,"commit":"fd773a9","metric":18.182,"metrics":{"passed_cases":2,"failed_cases":9,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774562027,"segment":5}
+{"run":16,"commit":"70cc59a","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"expand cli command surface","timestamp":1774562138,"segment":5}
+{"type":"config","name":"localllm-cli-semantic-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}

autoresearch.md

@@ -42,3 +42,6 @@ so later experiments can add real execution behavior without redesigning the UX
 - Segment 5 pivots autoresearch from template hardening to productization, specifically a complete CLI-first control surface on top of the current backend version.
 - The current repo has no CLI crate, no Tauri app, and no control-plane crate; the backend is still configured almost entirely by `LOCALLLM_*` environment variables and exposes only `/health`, `/v1/models`, and `/v1/chat/completions`.
 - The first benchmark for this segment is intentionally surface-oriented: ignored tests score whether the CLI presents the expected professional command set before deeper execution behavior is implemented.
+- Baseline run 15 (`fd773a9`) scored only `18.182%`: the new CLI scaffold currently exposes `status` and `models list`, but the broader management surface (`models scan/load/watch/download`, `auth`, `config`, `bench`, `logs`, `doctor`) is still absent.
+- Run 16 (`70cc59a`) expanded the Clap parser to the full intended command surface and raised segment 5 from `18.182%` to `100.0%` (11/11 passing), giving the CLI a stable UX contract before execution handlers are wired.
+- Segment 6 raises the bar from command presence to semantic usefulness: the benchmark now checks whether key workflows produce meaningful command summaries that preserve defaults and high-value arguments, as a stepping stone toward real execution planning.

crates/localllm-cli/src/main.rs

@@ -180,12 +180,16 @@ struct LogsArgs {
 
 fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
-    println!(
-        "CLI scaffold ready for {} at {}",
+    println!("CLI scaffold ready for {}", render_command_summary(&cli));
+    Ok(())
+}
+
+fn render_command_summary(cli: &Cli) -> String {
+    format!(
+        "command={} server={}",
         command_summary(&cli.command),
         cli.server
-    );
-    Ok(())
+    )
 }
 
 fn command_summary(command: &Commands) -> &'static str {
@@ -202,7 +206,7 @@ fn command_summary(command: &Commands) -> &'static str {
 
 #[cfg(test)]
 mod tests {
-    use super::Cli;
+    use super::{render_command_summary, AuthMode, Cli, Commands};
     use clap::Parser;
 
     #[test]
@@ -312,4 +316,90 @@ mod tests {
         let parsed = Cli::try_parse_from(["localllm", "doctor"]);
         assert!(parsed.is_ok());
     }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_status_summary_includes_output_contract() {
+        let cli = Cli::try_parse_from(["localllm", "--format", "json", "status"]).unwrap();
+        let summary = render_command_summary(&cli);
+
+        assert!(summary.contains("status"));
+        assert!(summary.contains("format=json"));
+        assert!(summary.contains("models_dir=./models"));
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_models_load_summary_includes_model_and_alias() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "models",
+            "load",
+            "./models/qwen.gguf",
+            "--alias",
+            "qwen-coder",
+        ])
+        .unwrap();
+        let summary = render_command_summary(&cli);
+
+        assert!(summary.contains("models.load"));
+        assert!(summary.contains("./models/qwen.gguf"));
+        assert!(summary.contains("alias=qwen-coder"));
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_auth_enable_summary_includes_mode_and_key_presence() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "auth",
+            "enable",
+            "--mode",
+            "local-only",
+            "--api-key",
+            "secret",
+        ])
+        .unwrap();
+        let summary = render_command_summary(&cli);
+
+        assert!(summary.contains("auth.enable"));
+        assert!(summary.contains("mode=local-only"));
+        assert!(summary.contains("has_api_key=true"));
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_bench_throughput_summary_includes_default_budget() {
+        let cli = Cli::try_parse_from(["localllm", "bench", "throughput"]).unwrap();
+        let summary = render_command_summary(&cli);
+
+        assert!(summary.contains("bench.throughput"));
+        assert!(summary.contains("samples=3"));
+        assert!(summary.contains("prompt_tokens=256"));
+        assert!(summary.contains("completion_tokens=128"));
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_logs_summary_includes_default_line_limit() {
+        let cli = Cli::try_parse_from(["localllm", "logs"]).unwrap();
+        let summary = render_command_summary(&cli);
+
+        assert!(summary.contains("logs"));
+        assert!(summary.contains("lines=200"));
+    }
+
+    #[test]
+    fn auth_mode_parser_accepts_local_only() {
+        let cli =
+            Cli::try_parse_from(["localllm", "auth", "enable", "--mode", "local-only"]).unwrap();
+
+        match cli.command {
+            Commands::Auth(auth) => match auth.command {
+                super::AuthSubcommand::Enable(args) => assert_eq!(args.mode, AuthMode::LocalOnly),
+                other => panic!("unexpected auth command: {:?}", other),
+            },
+            other => panic!("unexpected command: {:?}", other),
+        }
+    }
 }

experiments/worklog.md

@@ -144,3 +144,22 @@
 - Benchmark driver: `./autoresearch.sh` now runs ignored CLI-surface tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
 - Keep/discard rule is test-driven: higher `cli_surface_score` wins while the regular non-ignored suite remains green.
 - This segment starts by benchmarking the command surface itself before deeper execution and backend-admin semantics are implemented.
+
+### Run 15: baseline - cli_surface_score=18.182 (KEEP)
+- Timestamp: 2026-03-26 18:53
+- What changed: Initialized a new `crates/localllm-cli` crate with a minimal Clap-based CLI scaffold, switched the autoresearch harness from template tests to CLI surface tests, and defined the first ignored benchmark suite for professional command coverage.
+- Result: cli_surface_score=18.182, passed_cases=2, failed_cases=9, total_cases=11.
+- Insight: The scaffold already covers global flags plus `status` and `models list`, but almost the entire promised management surface is still missing.
+- Next: add the remaining top-level command groups and model/auth/config/bench/logs/doctor subcommands so the CLI surface matches the intended product contract before wiring execution internals.
+
+### Run 16: expand cli command surface - cli_surface_score=100.0 (KEEP)
+- Timestamp: 2026-03-26 18:55
+- What changed: Expanded `crates/localllm-cli/src/main.rs` into a broad Clap parser with top-level `status`, `models`, `auth`, `config`, `bench`, `logs`, and `doctor` commands plus the expected model/auth/config/throughput subcommands and global flags.
+- Result: cli_surface_score=100.0, passed_cases=11, failed_cases=0, total_cases=11.
+- Insight: The professional CLI contract can be locked in quickly at the parser layer before execution internals exist; Clap gives a maintainable foundation for the full management surface.
+- Next: raise the benchmark from command presence to semantic correctness, starting with command-specific arguments, defaults, and output-contract expectations for the highest-value workflows.
+
+## Segment 6: CLI Semantic Contracts
+- New objective: keep the CLI surface stable, but raise the benchmark so key commands preserve useful semantic detail such as defaults, high-value arguments, and summary/intention contracts.
+- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.