chore: expand cli backend response coverage

Cargo.lock

@@ -20,6 +20,56 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anstream"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
+[[package]]
+name = "anstyle-parse"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
+dependencies = [
+ "anstyle",
+ "once_cell_polyfill",
+ "windows-sys",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.102"
@@ -182,6 +232,52 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "clap"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
@@ -539,6 +635,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -593,6 +695,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "localllm-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.14"
@@ -681,6 +793,12 @@ version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -919,6 +1037,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "syn"
 version = "2.0.117"
@@ -1154,6 +1278,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.22.0"

autoresearch-dashboard.md

@@ -1,9 +1,10 @@
-# Autoresearch Dashboard: localllm-cli-execution-contracts
+# Autoresearch Dashboard: localllm-cli-backend-requests
 
-**Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0
+**Runs:** 2 | **Kept:** 2 | **Discarded:** 0 | **Crashed:** 0
-**Baseline:** cli_surface_score: 76.667% (#21)
+**Baseline:** cli_surface_score: 94.444% (#31)
-**Best:** cli_surface_score: 76.667% (#21, +0.0%)
+**Best:** cli_surface_score: 100.0% (#32, +5.9%)
 
 | # | commit | cli_surface_score | status | description |
 |---|--------|-------------------|--------|-------------|
-| 21 | 3d1460b | 76.667% (+0.0%) | keep | baseline |
+| 31 | f970c05 | 94.444% (+0.0%) | keep | baseline |
+| 32 | 5de35c5 | 100.0% (+5.9%) | keep | add cli backend request specs |

autoresearch.jsonl

@@ -39,3 +39,9 @@
 {"run":27,"commit":"7738acf","metric":87.5,"metrics":{"passed_cases":42,"failed_cases":6,"total_cases":48},"status":"keep","description":"baseline","timestamp":1774564340,"segment":11}
 {"run":28,"commit":"99e5041","metric":100.0,"metrics":{"passed_cases":48,"failed_cases":0,"total_cases":48},"status":"keep","description":"render cli dry-run previews","timestamp":1774564465,"segment":11}
 {"type":"config","name":"localllm-cli-backend-invocations","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
+{"run":29,"commit":"1ff5ba3","metric":94.118,"metrics":{"passed_cases":48,"failed_cases":3,"total_cases":51},"status":"keep","description":"baseline","timestamp":1774564791,"segment":12}
+{"run":30,"commit":"bf40b2f","metric":100.0,"metrics":{"passed_cases":51,"failed_cases":0,"total_cases":51},"status":"keep","description":"add cli backend invocation specs","timestamp":1774564898,"segment":12}
+{"type":"config","name":"localllm-cli-backend-requests","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
+{"run":31,"commit":"f970c05","metric":94.444,"metrics":{"passed_cases":51,"failed_cases":3,"total_cases":54},"status":"keep","description":"baseline","timestamp":1774565247,"segment":13}
+{"run":32,"commit":"5de35c5","metric":100.0,"metrics":{"passed_cases":54,"failed_cases":0,"total_cases":54},"status":"keep","description":"add cli backend request specs","timestamp":1774565368,"segment":13}
+{"type":"config","name":"localllm-cli-backend-responses","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}

autoresearch.md

@@ -63,3 +63,9 @@ so later experiments can add real execution behavior without redesigning the UX
 - Baseline run 27 (`7738acf`) scored `87.5%`: the CLI already had parser/summary/plan/contract/stub/preview layers, but `render_execution_preview()` still collapsed everything to `DRY RUN {summary}` instead of final text/json dry-run output.
 - Run 28 (`99e5041`) rendered the highest-value `ExecutionPreview` values into final user-facing text/json dry-run output, raising segment 11 from `87.5%` to `100.0%` (48/48 passing).
 - Segment 12 raises the bar from dry-run UX to the first real backend-call specifications for commands that can already work today, specifically `status`, `models list`, and `bench throughput`.
+- Baseline run 29 (`1ff5ba3`) scored `94.118%`: every earlier CLI layer still passed, but `backend_invocation()` was still a placeholder and the new backend-facing tests for live status/models/bench requests failed.
+- Run 30 (`bf40b2f`) added concrete backend invocation specs for `GET /health`, `GET /v1/models`, and `POST /v1/chat/completions` with preserved `--model`, token budget, server base URL, and auth presence, raising segment 12 from `94.118%` to `100.0%` (51/51 passing).
+- Segment 13 raises the bar again from backend invocation specs to executable backend request construction, checking whether the same live-compatible commands can now produce final HTTP method/url/header/body request objects with deterministic JSON payloads.
+- Run 31 (`f970c05`) scored `94.444%` because `backend_request()` was still a placeholder and the new backend-request tests failed for `status`, `models list`, and `bench throughput`.
+- Run 32 (`5de35c5`) introduced concrete executable backend request specs with bearer/content-type headers and deterministic JSON benchmark payloads, raising segment 13 from `94.444%` to `100.0%` (54/54 passing).
+- Segment 14 raises the bar from executable backend request specs to backend response interpretation, checking whether the same live-compatible commands can parse real response bodies into CLI-facing previews for health, model lists, and throughput usage metrics.

crates/localllm-cli/Cargo.toml

@@ -12,4 +12,5 @@ path = "src/main.rs"
 [dependencies]
 anyhow.workspace = true
 clap.workspace = true
+serde.workspace = true
 serde_json.workspace = true

crates/localllm-cli/src/main.rs

@@ -1,4 +1,5 @@
 use clap::{Args, Parser, Subcommand, ValueEnum};
+use serde::Deserialize;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
 enum OutputFormat {
@@ -321,6 +322,57 @@ enum BackendRequest {
     },
 }
 
+#[derive(Debug, Clone, PartialEq)]
+enum BackendResponsePreview {
+    SummaryOnly {
+        command: String,
+    },
+    Health {
+        status: String,
+        model_loaded: bool,
+        model_id: Option<String>,
+        uptime_secs: u64,
+    },
+    Models {
+        count: usize,
+        ids: Vec<String>,
+    },
+    Throughput {
+        model: String,
+        total_tokens: u32,
+        tokens_per_second: f64,
+    },
+}
+
+#[derive(Debug, Deserialize)]
+struct CliHealthResponse {
+    status: String,
+    model_loaded: bool,
+    model_id: Option<String>,
+    uptime_secs: u64,
+}
+
+#[derive(Debug, Deserialize)]
+struct CliModelList {
+    data: Vec<CliModelInfo>,
+}
+
+#[derive(Debug, Deserialize)]
+struct CliModelInfo {
+    id: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct CliBenchResponse {
+    usage: CliUsage,
+}
+
+#[derive(Debug, Deserialize)]
+struct CliUsage {
+    total_tokens: u32,
+    tokens_per_second: f64,
+}
+
 fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
     if cli.dry_run {
@@ -695,6 +747,33 @@ fn backend_request(cli: &Cli) -> BackendRequest {
     }
 }
 
+fn backend_response_preview(cli: &Cli, _status_code: u16, _body: &str) -> BackendResponsePreview {
+    match &cli.command {
+        Commands::Status => BackendResponsePreview::SummaryOnly {
+            command: "response-unimplemented".to_owned(),
+        },
+        Commands::Models(models) => match &models.command {
+            ModelsSubcommand::List => BackendResponsePreview::SummaryOnly {
+                command: "response-unimplemented".to_owned(),
+            },
+            _ => BackendResponsePreview::SummaryOnly {
+                command: command_summary(&cli.command),
+            },
+        },
+        Commands::Bench(bench) => match &bench.command {
+            BenchSubcommand::Throughput(_) => BackendResponsePreview::SummaryOnly {
+                command: "response-unimplemented".to_owned(),
+            },
+            _ => BackendResponsePreview::SummaryOnly {
+                command: command_summary(&cli.command),
+            },
+        },
+        _ => BackendResponsePreview::SummaryOnly {
+            command: command_summary(&cli.command),
+        },
+    }
+}
+
 fn bearer_headers(cli: &Cli) -> Vec<(String, String)> {
     cli.api_key
         .as_ref()
@@ -822,10 +901,11 @@ fn bench_command_summary(command: &BenchCommand) -> String {
 #[cfg(test)]
 mod tests {
     use super::{
-        backend_invocation, backend_request, execution_contract, execution_preview, execution_stub,
+        backend_invocation, backend_request, backend_response_preview, execution_contract,
-        plan_command, render_command_summary, render_execution_preview, AuthMode,
+        execution_preview, execution_stub, plan_command, render_command_summary,
-        BackendInvocation, BackendRequest, Cli, CommandPlan, Commands, ExecutionContract,
+        render_execution_preview, AuthMode, BackendInvocation, BackendRequest,
-        ExecutionPreview, ExecutionStub,
+        BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, ExecutionPreview,
+        ExecutionStub,
     };
     use clap::Parser;
     use serde_json::json;
@@ -1874,4 +1954,74 @@ mod tests {
             }
         );
     }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_status_response_parses_health_payload() {
+        let cli = Cli::try_parse_from(["localllm", "status"]).unwrap();
+
+        assert_eq!(
+            backend_response_preview(
+                &cli,
+                200,
+                r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#,
+            ),
+            BackendResponsePreview::Health {
+                status: "ok".to_owned(),
+                model_loaded: true,
+                model_id: Some("qwen-coder".to_owned()),
+                uptime_secs: 12,
+            }
+        );
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_models_list_response_collects_ids() {
+        let cli = Cli::try_parse_from(["localllm", "models", "list"]).unwrap();
+
+        assert_eq!(
+            backend_response_preview(
+                &cli,
+                200,
+                r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#,
+            ),
+            BackendResponsePreview::Models {
+                count: 2,
+                ids: vec!["qwen-coder".to_owned(), "gemma-3".to_owned()],
+            }
+        );
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_bench_response_reads_usage_metrics() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "bench",
+            "throughput",
+            "--model",
+            "qwen-coder",
+            "--samples",
+            "5",
+            "--prompt-tokens",
+            "8",
+            "--completion-tokens",
+            "64",
+        ])
+        .unwrap();
+
+        assert_eq!(
+            backend_response_preview(
+                &cli,
+                200,
+                r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#,
+            ),
+            BackendResponsePreview::Throughput {
+                model: "qwen-coder".to_owned(),
+                total_tokens: 72,
+                tokens_per_second: 42.5,
+            }
+        );
+    }
 }

experiments/worklog.md

@@ -277,3 +277,41 @@
 - New objective: raise the CLI benchmark from dry-run rendering to the first real backend-call specifications for commands that can already work today, especially `status`, `models list`, and `bench throughput`.
 - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
 - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
+
+## Segment 13: CLI Backend Requests
+- New objective: raise the CLI benchmark from backend invocation specs to executable backend request construction for the commands that already align with live server endpoints today.
+- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
+
+### Run 29: baseline - cli_surface_score=94.118 (KEEP)
+- Timestamp: 2026-03-26 19:39
+- What changed: Expanded the benchmark from final dry-run rendering to backend invocation specs for commands that can already work today, adding ignored tests for live `GET /health`, live `GET /v1/models`, and a throughput `POST /v1/chat/completions` request that preserves the selected model and token budget.
+- Result: cli_surface_score=94.118, passed_cases=48, failed_cases=3, total_cases=51.
+- Insight: The CLI already handled every previous parser/planning/dry-run layer, but `backend_invocation()` was still a placeholder, and `models list` had not yet been promoted into the real backend-facing seam.
+- Next: implement concrete backend invocation specs for `status`, `models list`, and `bench throughput` using the resolved server base URL and auth presence.
+
+### Run 30: add cli backend invocation specs - cli_surface_score=100.0 (KEEP)
+- Timestamp: 2026-03-26 19:41
+- What changed: Implemented `backend_invocation(&Cli)` in `crates/localllm-cli/src/main.rs` so `status` maps to live `GET /health`, `models list` maps to live `GET /v1/models`, and `bench throughput` maps to a live `POST /v1/chat/completions` invocation that preserves `--model`, samples, token budget, and auth presence.
+- Result: cli_surface_score=100.0, passed_cases=51, failed_cases=0, total_cases=51.
+- Insight: The CLI now crosses from dry-run-only UX into concrete backend-facing invocation specs for commands that can already work against the current server today, without needing the future control-plane endpoints yet.
+- Next: raise the bar from backend invocation specs to actual request payload construction or the first executable handlers for these already-supported commands.
+
+### Run 31: baseline - cli_surface_score=94.444 (KEEP)
+- Timestamp: 2026-03-26 19:47
+- What changed: Expanded the benchmark from backend invocation specs to executable backend request construction, requiring final HTTP method/url/header/body request objects with deterministic JSON payloads for `status`, `models list`, and `bench throughput`.
+- Result: cli_surface_score=94.444, passed_cases=51, failed_cases=3, total_cases=54.
+- Insight: The CLI already had parser, summaries, plans, contracts, stubs, previews, dry-run rendering, and invocation specs, but `backend_request(&Cli)` still returned `SummaryOnly { command: "request-unimplemented" }` instead of final request objects.
+- Next: map `BackendInvocation::HttpCall` into executable `BackendRequest::HttpRequest` values with bearer headers, JSON content-type, and deterministic benchmark payload generation.
+
+### Run 32: add cli backend request specs - cli_surface_score=100.0 (KEEP)
+- Timestamp: 2026-03-26 19:49
+- What changed: Implemented `backend_request(&Cli)` in `crates/localllm-cli/src/main.rs` plus helpers for bearer headers, JSON headers, deterministic benchmark prompt construction, and exact chat payload generation for throughput requests.
+- Result: cli_surface_score=100.0, passed_cases=54, failed_cases=0, total_cases=54.
+- Insight: The first live-compatible CLI workflows now preserve the full executable backend request boundary - method, URL, auth headers, content-type, and deterministic JSON body - while still remaining execution-free.
+- Next: raise the bar from request construction to the first live response-handling or executable handler seams for the same workflows.
+
+## Segment 14: CLI Backend Responses
+- New objective: raise the CLI benchmark from executable backend request construction to backend response interpretation for the same live-compatible workflows, starting with `status`, `models list`, and `bench throughput`.
+- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.