From bf985ea9eaa688361899450f1573daacf8c5c631 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 20:01:09 -0300 Subject: [PATCH] chore: expand cli backend response coverage --- Cargo.lock | 130 ++++++++++++++++++++++++++ autoresearch-dashboard.md | 11 ++- autoresearch.jsonl | 6 ++ autoresearch.md | 6 ++ crates/localllm-cli/Cargo.toml | 1 + crates/localllm-cli/src/main.rs | 158 +++++++++++++++++++++++++++++++- experiments/worklog.md | 38 ++++++++ 7 files changed, 341 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 994b807..a1b6cb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -182,6 +232,52 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -539,6 +635,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -593,6 +695,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "localllm-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -681,6 +793,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "parking_lot" version = "0.12.5" @@ -919,6 +1037,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.117" @@ -1154,6 +1278,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.22.0" diff --git a/autoresearch-dashboard.md b/autoresearch-dashboard.md index 728d54b..1138621 100644 --- a/autoresearch-dashboard.md +++ b/autoresearch-dashboard.md @@ -1,9 +1,10 @@ -# Autoresearch Dashboard: localllm-cli-execution-contracts +# Autoresearch Dashboard: localllm-cli-backend-requests -**Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0 -**Baseline:** cli_surface_score: 76.667% (#21) -**Best:** cli_surface_score: 76.667% (#21, +0.0%) +**Runs:** 2 | **Kept:** 2 | **Discarded:** 0 | **Crashed:** 0 +**Baseline:** cli_surface_score: 94.444% (#31) +**Best:** cli_surface_score: 100.0% (#32, +5.9%) | # | commit | cli_surface_score | status | description | |---|--------|-------------------|--------|-------------| -| 21 | 3d1460b | 76.667% (+0.0%) | keep | baseline | +| 31 | f970c05 | 94.444% (+0.0%) | keep | baseline | +| 32 | 5de35c5 | 100.0% (+5.9%) | keep | add cli backend request specs | diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 9f3324b..e508a1e 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -39,3 +39,9 @@ {"run":27,"commit":"7738acf","metric":87.5,"metrics":{"passed_cases":42,"failed_cases":6,"total_cases":48},"status":"keep","description":"baseline","timestamp":1774564340,"segment":11} {"run":28,"commit":"99e5041","metric":100.0,"metrics":{"passed_cases":48,"failed_cases":0,"total_cases":48},"status":"keep","description":"render cli dry-run previews","timestamp":1774564465,"segment":11} {"type":"config","name":"localllm-cli-backend-invocations","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":29,"commit":"1ff5ba3","metric":94.118,"metrics":{"passed_cases":48,"failed_cases":3,"total_cases":51},"status":"keep","description":"baseline","timestamp":1774564791,"segment":12} +{"run":30,"commit":"bf40b2f","metric":100.0,"metrics":{"passed_cases":51,"failed_cases":0,"total_cases":51},"status":"keep","description":"add cli backend invocation specs","timestamp":1774564898,"segment":12} +{"type":"config","name":"localllm-cli-backend-requests","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":31,"commit":"f970c05","metric":94.444,"metrics":{"passed_cases":51,"failed_cases":3,"total_cases":54},"status":"keep","description":"baseline","timestamp":1774565247,"segment":13} +{"run":32,"commit":"5de35c5","metric":100.0,"metrics":{"passed_cases":54,"failed_cases":0,"total_cases":54},"status":"keep","description":"add cli backend request specs","timestamp":1774565368,"segment":13} +{"type":"config","name":"localllm-cli-backend-responses","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index 9e476d0..72fb729 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -63,3 +63,9 @@ so later experiments can add real execution behavior without redesigning the UX - Baseline run 27 (`7738acf`) scored `87.5%`: the CLI already had parser/summary/plan/contract/stub/preview layers, but `render_execution_preview()` still collapsed everything to `DRY RUN {summary}` instead of final text/json dry-run output. - Run 28 (`99e5041`) rendered the highest-value `ExecutionPreview` values into final user-facing text/json dry-run output, raising segment 11 from `87.5%` to `100.0%` (48/48 passing). - Segment 12 raises the bar from dry-run UX to the first real backend-call specifications for commands that can already work today, specifically `status`, `models list`, and `bench throughput`. +- Baseline run 29 (`1ff5ba3`) scored `94.118%`: every earlier CLI layer still passed, but `backend_invocation()` was still a placeholder and the new backend-facing tests for live status/models/bench requests failed. +- Run 30 (`bf40b2f`) added concrete backend invocation specs for `GET /health`, `GET /v1/models`, and `POST /v1/chat/completions` with preserved `--model`, token budget, server base URL, and auth presence, raising segment 12 from `94.118%` to `100.0%` (51/51 passing). +- Segment 13 raises the bar again from backend invocation specs to executable backend request construction, checking whether the same live-compatible commands can now produce final HTTP method/url/header/body request objects with deterministic JSON payloads. +- Run 31 (`f970c05`) scored `94.444%` because `backend_request()` was still a placeholder and the new backend-request tests failed for `status`, `models list`, and `bench throughput`. +- Run 32 (`5de35c5`) introduced concrete executable backend request specs with bearer/content-type headers and deterministic JSON benchmark payloads, raising segment 13 from `94.444%` to `100.0%` (54/54 passing). +- Segment 14 raises the bar from executable backend request specs to backend response interpretation, checking whether the same live-compatible commands can parse real response bodies into CLI-facing previews for health, model lists, and throughput usage metrics. diff --git a/crates/localllm-cli/Cargo.toml b/crates/localllm-cli/Cargo.toml index f50f6ee..5df3fbe 100644 --- a/crates/localllm-cli/Cargo.toml +++ b/crates/localllm-cli/Cargo.toml @@ -12,4 +12,5 @@ path = "src/main.rs" [dependencies] anyhow.workspace = true clap.workspace = true +serde.workspace = true serde_json.workspace = true diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index 6b3504a..53caa52 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -1,4 +1,5 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; +use serde::Deserialize; #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] enum OutputFormat { @@ -321,6 +322,57 @@ enum BackendRequest { }, } +#[derive(Debug, Clone, PartialEq)] +enum BackendResponsePreview { + SummaryOnly { + command: String, + }, + Health { + status: String, + model_loaded: bool, + model_id: Option, + uptime_secs: u64, + }, + Models { + count: usize, + ids: Vec, + }, + Throughput { + model: String, + total_tokens: u32, + tokens_per_second: f64, + }, +} + +#[derive(Debug, Deserialize)] +struct CliHealthResponse { + status: String, + model_loaded: bool, + model_id: Option, + uptime_secs: u64, +} + +#[derive(Debug, Deserialize)] +struct CliModelList { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct CliModelInfo { + id: String, +} + +#[derive(Debug, Deserialize)] +struct CliBenchResponse { + usage: CliUsage, +} + +#[derive(Debug, Deserialize)] +struct CliUsage { + total_tokens: u32, + tokens_per_second: f64, +} + fn main() -> anyhow::Result<()> { let cli = Cli::parse(); if cli.dry_run { @@ -695,6 +747,33 @@ fn backend_request(cli: &Cli) -> BackendRequest { } } +fn backend_response_preview(cli: &Cli, _status_code: u16, _body: &str) -> BackendResponsePreview { + match &cli.command { + Commands::Status => BackendResponsePreview::SummaryOnly { + command: "response-unimplemented".to_owned(), + }, + Commands::Models(models) => match &models.command { + ModelsSubcommand::List => BackendResponsePreview::SummaryOnly { + command: "response-unimplemented".to_owned(), + }, + _ => BackendResponsePreview::SummaryOnly { + command: command_summary(&cli.command), + }, + }, + Commands::Bench(bench) => match &bench.command { + BenchSubcommand::Throughput(_) => BackendResponsePreview::SummaryOnly { + command: "response-unimplemented".to_owned(), + }, + _ => BackendResponsePreview::SummaryOnly { + command: command_summary(&cli.command), + }, + }, + _ => BackendResponsePreview::SummaryOnly { + command: command_summary(&cli.command), + }, + } +} + fn bearer_headers(cli: &Cli) -> Vec<(String, String)> { cli.api_key .as_ref() @@ -822,10 +901,11 @@ fn bench_command_summary(command: &BenchCommand) -> String { #[cfg(test)] mod tests { use super::{ - backend_invocation, backend_request, execution_contract, execution_preview, execution_stub, - plan_command, render_command_summary, render_execution_preview, AuthMode, - BackendInvocation, BackendRequest, Cli, CommandPlan, Commands, ExecutionContract, - ExecutionPreview, ExecutionStub, + backend_invocation, backend_request, backend_response_preview, execution_contract, + execution_preview, execution_stub, plan_command, render_command_summary, + render_execution_preview, AuthMode, BackendInvocation, BackendRequest, + BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, ExecutionPreview, + ExecutionStub, }; use clap::Parser; use serde_json::json; @@ -1874,4 +1954,74 @@ mod tests { } ); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_response_parses_health_payload() { + let cli = Cli::try_parse_from(["localllm", "status"]).unwrap(); + + assert_eq!( + backend_response_preview( + &cli, + 200, + r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#, + ), + BackendResponsePreview::Health { + status: "ok".to_owned(), + model_loaded: true, + model_id: Some("qwen-coder".to_owned()), + uptime_secs: 12, + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_list_response_collects_ids() { + let cli = Cli::try_parse_from(["localllm", "models", "list"]).unwrap(); + + assert_eq!( + backend_response_preview( + &cli, + 200, + r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#, + ), + BackendResponsePreview::Models { + count: 2, + ids: vec!["qwen-coder".to_owned(), "gemma-3".to_owned()], + } + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_response_reads_usage_metrics() { + let cli = Cli::try_parse_from([ + "localllm", + "bench", + "throughput", + "--model", + "qwen-coder", + "--samples", + "5", + "--prompt-tokens", + "8", + "--completion-tokens", + "64", + ]) + .unwrap(); + + assert_eq!( + backend_response_preview( + &cli, + 200, + r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#, + ), + BackendResponsePreview::Throughput { + model: "qwen-coder".to_owned(), + total_tokens: 72, + tokens_per_second: 42.5, + } + ); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index 68fa4ce..d43c87a 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -277,3 +277,41 @@ - New objective: raise the CLI benchmark from dry-run rendering to the first real backend-call specifications for commands that can already work today, especially `status`, `models list`, and `bench throughput`. - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +## Segment 13: CLI Backend Requests +- New objective: raise the CLI benchmark from backend invocation specs to executable backend request construction for the commands that already align with live server endpoints today. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 29: baseline - cli_surface_score=94.118 (KEEP) +- Timestamp: 2026-03-26 19:39 +- What changed: Expanded the benchmark from final dry-run rendering to backend invocation specs for commands that can already work today, adding ignored tests for live `GET /health`, live `GET /v1/models`, and a throughput `POST /v1/chat/completions` request that preserves the selected model and token budget. +- Result: cli_surface_score=94.118, passed_cases=48, failed_cases=3, total_cases=51. +- Insight: The CLI already handled every previous parser/planning/dry-run layer, but `backend_invocation()` was still a placeholder, and `models list` had not yet been promoted into the real backend-facing seam. +- Next: implement concrete backend invocation specs for `status`, `models list`, and `bench throughput` using the resolved server base URL and auth presence. + +### Run 30: add cli backend invocation specs - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:41 +- What changed: Implemented `backend_invocation(&Cli)` in `crates/localllm-cli/src/main.rs` so `status` maps to live `GET /health`, `models list` maps to live `GET /v1/models`, and `bench throughput` maps to a live `POST /v1/chat/completions` invocation that preserves `--model`, samples, token budget, and auth presence. +- Result: cli_surface_score=100.0, passed_cases=51, failed_cases=0, total_cases=51. +- Insight: The CLI now crosses from dry-run-only UX into concrete backend-facing invocation specs for commands that can already work against the current server today, without needing the future control-plane endpoints yet. +- Next: raise the bar from backend invocation specs to actual request payload construction or the first executable handlers for these already-supported commands. + +### Run 31: baseline - cli_surface_score=94.444 (KEEP) +- Timestamp: 2026-03-26 19:47 +- What changed: Expanded the benchmark from backend invocation specs to executable backend request construction, requiring final HTTP method/url/header/body request objects with deterministic JSON payloads for `status`, `models list`, and `bench throughput`. +- Result: cli_surface_score=94.444, passed_cases=51, failed_cases=3, total_cases=54. +- Insight: The CLI already had parser, summaries, plans, contracts, stubs, previews, dry-run rendering, and invocation specs, but `backend_request(&Cli)` still returned `SummaryOnly { command: "request-unimplemented" }` instead of final request objects. +- Next: map `BackendInvocation::HttpCall` into executable `BackendRequest::HttpRequest` values with bearer headers, JSON content-type, and deterministic benchmark payload generation. + +### Run 32: add cli backend request specs - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:49 +- What changed: Implemented `backend_request(&Cli)` in `crates/localllm-cli/src/main.rs` plus helpers for bearer headers, JSON headers, deterministic benchmark prompt construction, and exact chat payload generation for throughput requests. +- Result: cli_surface_score=100.0, passed_cases=54, failed_cases=0, total_cases=54. +- Insight: The first live-compatible CLI workflows now preserve the full executable backend request boundary - method, URL, auth headers, content-type, and deterministic JSON body - while still remaining execution-free. +- Next: raise the bar from request construction to the first live response-handling or executable handler seams for the same workflows. + +## Segment 14: CLI Backend Responses +- New objective: raise the CLI benchmark from executable backend request construction to backend response interpretation for the same live-compatible workflows, starting with `status`, `models list`, and `bench throughput`. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.