From 1881b63d7b22c5ca284ccba3237093bbb1c65dfb Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 19:10:55 -0300 Subject: [PATCH] chore: expand cli execution contract coverage --- autoresearch.md | 2 + crates/localllm-cli/src/main.rs | 185 +++++++++++++++++++++++++++++++- experiments/worklog.md | 12 +++ 3 files changed, 198 insertions(+), 1 deletion(-) diff --git a/autoresearch.md b/autoresearch.md index 9909677..8efcd3b 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -49,3 +49,5 @@ so later experiments can add real execution behavior without redesigning the UX - Run 18 (`66b30de`) enriched semantic summaries across the CLI and raised segment 6 from `68.75%` to `100.0%` (16/16 passing), so the next bar should move from summary correctness to execution-planning or output-contract semantics. - Segment 7 raises the bar from semantic summaries to structured execution plans, checking whether key commands preserve actionable details in a machine-usable plan that can later drive real handlers. - Baseline run 19 (`9154a7b`) scored `69.565%`: the CLI still passes all parser and summary tests, but every new plan-oriented workflow fails because `plan_command()` currently returns `SummaryOnly` for all commands. +- Run 20 (`bc584be`) introduced concrete `CommandPlan` variants for status, model scan/load, auth enable, config set, throughput bench, and logs, raising segment 7 from `69.565%` to `100.0%` (23/23 passing). +- Segment 8 raises the bar from machine-usable plans to execution contracts, checking whether key workflows can already describe the concrete HTTP/local action they intend to perform at the backend/control-plane seam. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index e2186cf..fff03da 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -216,6 +216,43 @@ enum CommandPlan { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +enum ExecutionContract { + SummaryOnly { + command: String, + }, + HttpGet { + path: String, + api_key_present: bool, + }, + LocalModelScan { + models_dir: String, + }, + LocalModelLoad { + models_dir: String, + model: String, + alias: Option, + }, + LocalAuthConfig { + mode: AuthMode, + api_key: Option, + }, + LocalConfigWrite { + key: String, + value: String, + }, + ThroughputRequest { + path: String, + api_key_present: bool, + samples: u32, + prompt_tokens: u32, + completion_tokens: u32, + }, + LocalLogRead { + lines: u32, + }, +} + fn main() -> anyhow::Result<()> { let cli = Cli::parse(); println!("CLI scaffold ready for {}", render_command_summary(&cli)); @@ -289,6 +326,17 @@ fn plan_command(cli: &Cli) -> CommandPlan { } } +fn execution_contract(plan: &CommandPlan) -> ExecutionContract { + match plan { + CommandPlan::SummaryOnly { command } => ExecutionContract::SummaryOnly { + command: command.clone(), + }, + _ => ExecutionContract::SummaryOnly { + command: "unplanned".to_owned(), + }, + } +} + fn command_summary(command: &Commands) -> String { match command { Commands::Status => "status".to_owned(), @@ -374,7 +422,10 @@ fn bench_command_summary(command: &BenchCommand) -> String { #[cfg(test)] mod tests { - use super::{plan_command, render_command_summary, AuthMode, Cli, CommandPlan, Commands}; + use super::{ + execution_contract, plan_command, render_command_summary, AuthMode, Cli, CommandPlan, + Commands, ExecutionContract, + }; use clap::Parser; #[test] @@ -700,4 +751,136 @@ mod tests { assert_eq!(plan_command(&cli), CommandPlan::ReadLogs { lines: 50 }); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_contract_targets_health_endpoint() { + let cli = Cli::try_parse_from(["localllm", "--api-key", "secret", "status"]).unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::HttpGet { + path: "/health".to_owned(), + api_key_present: true, + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_scan_contract_targets_local_models_dir() { + let cli = Cli::try_parse_from(["localllm", "--models-dir", "D:/models", "models", "scan"]) + .unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::LocalModelScan { + models_dir: "D:/models".to_owned(), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_load_contract_preserves_local_load_details() { + let cli = Cli::try_parse_from([ + "localllm", + "--models-dir", + "D:/models", + "models", + "load", + "qwen.gguf", + "--alias", + "qwen-coder", + ]) + .unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::LocalModelLoad { + models_dir: "D:/models".to_owned(), + model: "qwen.gguf".to_owned(), + alias: Some("qwen-coder".to_owned()), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_auth_enable_contract_preserves_local_auth_change() { + let cli = Cli::try_parse_from([ + "localllm", + "auth", + "enable", + "--mode", + "local-only", + "--api-key", + "secret", + ]) + .unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::LocalAuthConfig { + mode: AuthMode::LocalOnly, + api_key: Some("secret".to_owned()), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_config_set_contract_preserves_local_write() { + let cli = Cli::try_parse_from(["localllm", "config", "set", "default_model", "qwen-coder"]) + .unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::LocalConfigWrite { + key: "default_model".to_owned(), + value: "qwen-coder".to_owned(), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_throughput_contract_targets_chat_endpoint() { + let cli = Cli::try_parse_from([ + "localllm", + "--api-key", + "secret", + "bench", + "throughput", + "--samples", + "5", + "--prompt-tokens", + "512", + "--completion-tokens", + "64", + ]) + .unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::ThroughputRequest { + path: "/v1/chat/completions".to_owned(), + api_key_present: true, + samples: 5, + prompt_tokens: 512, + completion_tokens: 64, + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_logs_contract_preserves_local_read() { + let cli = Cli::try_parse_from(["localllm", "logs", "--lines", "50"]).unwrap(); + + assert_eq!( + execution_contract(&plan_command(&cli)), + ExecutionContract::LocalLogRead { lines: 50 } + ); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index b154165..5472f94 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -176,6 +176,18 @@ - Insight: The semantic summary layer is already solid, but `plan_command()` is still a stub that collapses everything to `SummaryOnly`, so execution planning has not been wired yet. - Next: implement concrete `CommandPlan` mapping for status, models scan/load, auth enable, config set, bench throughput, and logs. +### Run 20: add cli execution plans - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:05 +- What changed: Implemented concrete `CommandPlan` mapping in `crates/localllm-cli/src/main.rs` for status, model scan/load, auth enable, config set, throughput bench, and logs while leaving lower-priority commands on `SummaryOnly`. +- Result: cli_surface_score=100.0, passed_cases=23, failed_cases=0, total_cases=23. +- Insight: The CLI now has a machine-usable planning layer for the highest-value workflows, which is a stronger base for later real handlers and Tauri reuse than summary strings alone. +- Next: raise the benchmark from structured plans to execution-ready request payloads or output contracts at the backend/control-plane seam. + +## Segment 8: CLI Execution Contracts +- New objective: turn the highest-value `CommandPlan` variants into concrete execution contracts that describe the exact HTTP or local operation each workflow intends to perform. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + ### Run 17: baseline - cli_surface_score=68.75 (KEEP) - Timestamp: 2026-03-26 18:59 - What changed: Raised the benchmark with summary-focused ignored tests that require `render_command_summary()` to preserve output format, models directory, command-specific arguments, and key defaults for status, models load, auth enable, bench throughput, and logs.