From 1454e64066e279d1fb8acc2463b1c823c6da253d Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 19:19:29 -0300 Subject: [PATCH] chore: expand cli execution stub coverage --- autoresearch.jsonl | 2 + autoresearch.md | 2 + crates/localllm-cli/src/main.rs | 160 +++++++++++++++++++++++++++++++- experiments/worklog.md | 12 +++ 4 files changed, 174 insertions(+), 2 deletions(-) diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 1e240eb..caa81e2 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -28,3 +28,5 @@ {"run":20,"commit":"bc584be","metric":100.0,"metrics":{"passed_cases":23,"failed_cases":0,"total_cases":23},"status":"keep","description":"add cli execution plans","timestamp":1774562759,"segment":7} {"type":"config","name":"localllm-cli-execution-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} {"run":21,"commit":"3d1460b","metric":76.667,"metrics":{"passed_cases":23,"failed_cases":7,"total_cases":30},"status":"keep","description":"baseline","timestamp":1774563087,"segment":8} +{"run":22,"commit":"38acb52","metric":100.0,"metrics":{"passed_cases":30,"failed_cases":0,"total_cases":30},"status":"keep","description":"add cli execution contracts","timestamp":1774563260,"segment":8} +{"type":"config","name":"localllm-cli-execution-stubs","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index b139ef0..bd0e75f 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -52,3 +52,5 @@ so later experiments can add real execution behavior without redesigning the UX - Run 20 (`bc584be`) introduced concrete `CommandPlan` variants for status, model scan/load, auth enable, config set, throughput bench, and logs, raising segment 7 from `69.565%` to `100.0%` (23/23 passing). - Segment 8 raises the bar from machine-usable plans to execution contracts, checking whether key workflows can already describe the concrete HTTP/local action they intend to perform at the backend/control-plane seam. - Baseline run 21 (`3d1460b`) scored `76.667%`: the CLI already preserves parser, summary, and plan semantics, but `execution_contract()` still collapses the new workflows to `SummaryOnly { command: "unplanned" }` instead of concrete HTTP/local contracts. +- Run 22 (`38acb52`) mapped the highest-value `CommandPlan` variants into concrete `ExecutionContract` values for `/health`, `/v1/chat/completions`, local model scan/load, local auth/config mutation, and local log reads, raising segment 8 from `76.667%` to `100.0%` (30/30 passing). +- Segment 9 raises the bar from contract shape to execution stubs, checking whether the same high-value workflows can already describe concrete HTTP method/path/body details or named local actions that a future real executor can run directly. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index 961e4fe..3d3a88c 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -253,6 +253,24 @@ enum ExecutionContract { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +enum ExecutionStub { + SummaryOnly { + command: String, + }, + HttpRequest { + method: &'static str, + path: String, + api_key_present: bool, + body: Option, + }, + LocalAction { + action: &'static str, + target: String, + detail: Option, + }, +} + fn main() -> anyhow::Result<()> { let cli = Cli::parse(); println!("CLI scaffold ready for {}", render_command_summary(&cli)); @@ -374,6 +392,17 @@ fn execution_contract(plan: &CommandPlan) -> ExecutionContract { } } +fn execution_stub(contract: &ExecutionContract) -> ExecutionStub { + match contract { + ExecutionContract::SummaryOnly { command } => ExecutionStub::SummaryOnly { + command: command.clone(), + }, + _ => ExecutionStub::SummaryOnly { + command: "unimplemented".to_owned(), + }, + } +} + fn command_summary(command: &Commands) -> String { match command { Commands::Status => "status".to_owned(), @@ -460,8 +489,8 @@ fn bench_command_summary(command: &BenchCommand) -> String { #[cfg(test)] mod tests { use super::{ - execution_contract, plan_command, render_command_summary, AuthMode, Cli, CommandPlan, - Commands, ExecutionContract, + execution_contract, execution_stub, plan_command, render_command_summary, AuthMode, Cli, + CommandPlan, Commands, ExecutionContract, ExecutionStub, }; use clap::Parser; @@ -920,4 +949,131 @@ mod tests { ExecutionContract::LocalLogRead { lines: 50 } ); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_stub_targets_http_get_health() { + let cli = Cli::try_parse_from(["localllm", "--api-key", "secret", "status"]).unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::HttpRequest { + method: "GET", + path: "/health".to_owned(), + api_key_present: true, + body: None, + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_scan_stub_names_local_scan_action() { + let cli = Cli::try_parse_from(["localllm", "--models-dir", "D:/models", "models", "scan"]) + .unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::LocalAction { + action: "scan_models_dir", + target: "D:/models".to_owned(), + detail: None, + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_load_stub_names_local_load_action() { + let cli = Cli::try_parse_from([ + "localllm", + "--models-dir", + "D:/models", + "models", + "load", + "qwen.gguf", + "--alias", + "qwen-coder", + ]) + .unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::LocalAction { + action: "load_model", + target: "D:/models".to_owned(), + detail: Some("model=qwen.gguf alias=qwen-coder".to_owned()), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_auth_enable_stub_names_local_auth_action() { + let cli = Cli::try_parse_from([ + "localllm", + "auth", + "enable", + "--mode", + "local-only", + "--api-key", + "secret", + ]) + .unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::LocalAction { + action: "configure_auth", + target: "auth".to_owned(), + detail: Some("mode=local-only has_api_key=true".to_owned()), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_stub_targets_chat_completion_post() { + let cli = Cli::try_parse_from([ + "localllm", + "--api-key", + "secret", + "bench", + "throughput", + "--samples", + "5", + "--prompt-tokens", + "512", + "--completion-tokens", + "64", + ]) + .unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::HttpRequest { + method: "POST", + path: "/v1/chat/completions".to_owned(), + api_key_present: true, + body: Some( + "samples=5 prompt_tokens=512 completion_tokens=64 stream=false".to_owned(), + ), + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_logs_stub_names_local_tail_action() { + let cli = Cli::try_parse_from(["localllm", "logs", "--lines", "50"]).unwrap(); + + assert_eq!( + execution_stub(&execution_contract(&plan_command(&cli))), + ExecutionStub::LocalAction { + action: "tail_logs", + target: "logs".to_owned(), + detail: Some("lines=50".to_owned()), + } + ); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index 1f887db..1feb396 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -195,6 +195,13 @@ - Insight: The planning layer is now solid, but `execution_contract()` is still stubbed and returns `SummaryOnly { command: "unplanned" }` for every concrete plan, so the CLI cannot yet describe its backend/control-plane seam accurately. - Next: map the high-value `CommandPlan` variants into concrete HTTP/local execution contracts for status, models scan/load, auth enable, config set, throughput bench, and logs. +### Run 22: add cli execution contracts - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:14 +- What changed: Implemented concrete `execution_contract()` mapping in `crates/localllm-cli/src/main.rs`, turning the highest-value `CommandPlan` variants into explicit HTTP or local contracts for health probing, model scan/load, auth/config mutation, throughput requests, and log reads. +- Result: cli_surface_score=100.0, passed_cases=30, failed_cases=0, total_cases=30. +- Insight: The CLI now has a clean progression from parser -> semantic summary -> machine-usable plan -> concrete execution contract for the highest-value workflows, which is a strong base for wiring real handlers next. +- Next: raise the benchmark from contract shape to actual execution stubs or backend request payload semantics, especially for status, throughput bench, and model operations. + ### Run 17: baseline - cli_surface_score=68.75 (KEEP) - Timestamp: 2026-03-26 18:59 - What changed: Raised the benchmark with summary-focused ignored tests that require `render_command_summary()` to preserve output format, models directory, command-specific arguments, and key defaults for status, models load, auth enable, bench throughput, and logs. @@ -208,3 +215,8 @@ - Result: cli_surface_score=100.0, passed_cases=16, failed_cases=0, total_cases=16. - Insight: A lightweight semantic/intention layer is enough to make the CLI contract meaningfully richer before wiring real handlers; the CLI now preserves actionable detail instead of collapsing everything to top-level groups. - Next: raise the benchmark again toward execution planning or output contracts, e.g. structured action plans for backend/admin operations rather than just human-readable summaries. + +## Segment 9: CLI Execution Stubs +- New objective: raise the CLI benchmark from abstract execution contracts to concrete execution stubs that preserve HTTP method/path/body semantics or named local actions for the highest-value workflows. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.