chore: expand cli live execution coverage

autoresearch.jsonl

@@ -48,3 +48,6 @@
 {"run":33,"commit":"8cf03e3","metric":94.737,"metrics":{"passed_cases":54,"failed_cases":3,"total_cases":57},"status":"keep","description":"baseline","timestamp":1774566069,"segment":14}
 {"run":34,"commit":"0506a4c","metric":100.0,"metrics":{"passed_cases":57,"failed_cases":0,"total_cases":57},"status":"keep","description":"add cli backend response previews","timestamp":1774566132,"segment":14}
 {"type":"config","name":"localllm-cli-live-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
+{"run":35,"commit":"e1ada2c","metric":90.476,"metrics":{"passed_cases":57,"failed_cases":6,"total_cases":63},"status":"keep","description":"baseline","timestamp":1774566405,"segment":15}
+{"run":36,"commit":"c54107d","metric":100.0,"metrics":{"passed_cases":63,"failed_cases":0,"total_cases":63},"status":"keep","description":"render cli live outputs","timestamp":1774566456,"segment":15}
+{"type":"config","name":"localllm-cli-live-execution","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}

autoresearch.md

@@ -72,3 +72,6 @@ so later experiments can add real execution behavior without redesigning the UX
 - Run 33 (`8cf03e3`) scored `94.737%` because `backend_response_preview()` was still a placeholder and the new response-parsing tests failed for `status`, `models list`, and `bench throughput`.
 - Run 34 (`0506a4c`) introduced CLI-local backend response previews for health payloads, model list ids/count, and throughput usage metrics, raising segment 14 from `94.737%` to `100.0%` (57/57 passing).
 - Segment 15 raises the bar from internal backend response previews to final user-facing live output rendering, checking whether the same live-compatible commands can render text/json output directly from parsed backend responses.
+- Run 35 (`e1ada2c`) scored `90.476%` because `render_backend_response()` still collapsed the live-compatible workflows to `LIVE {summary}` instead of final text/json output.
+- Run 36 (`c54107d`) rendered the first live-compatible backend responses into final user-facing text/json output, raising segment 15 from `90.476%` to `100.0%` (63/63 passing).
+- Segment 16 raises the bar from live output rendering to the first real executable handlers, checking whether the same live-compatible workflows can execute through an injectable transport seam and return the final rendered output end-to-end.

crates/localllm-cli/src/main.rs

@@ -344,6 +344,12 @@ enum BackendResponsePreview {
     },
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct LiveExecutionResponse {
+    status_code: u16,
+    body: String,
+}
+
 #[derive(Debug, Deserialize)]
 struct CliHealthResponse {
     status: String,
@@ -518,6 +524,21 @@ fn render_backend_response(cli: &Cli, status_code: u16, body: &str) -> String {
     }
 }
 
+fn execute_live_command_with<F>(cli: &Cli, mut transport: F) -> String
+where
+    F: FnMut(&BackendRequest) -> LiveExecutionResponse,
+{
+    let request = backend_request(cli);
+
+    match request {
+        BackendRequest::SummaryOnly { command } => format!("LIVE {}", command),
+        _ => {
+            let _response = transport(&request);
+            format!("LIVE EXEC {}", render_command_summary(cli))
+        }
+    }
+}
+
 fn render_json_optional_string(value: Option<&str>) -> String {
     match value {
         Some(value) => format!(r#""{}""#, value),
@@ -999,11 +1020,11 @@ fn bench_command_summary(command: &BenchCommand) -> String {
 #[cfg(test)]
 mod tests {
     use super::{
-        backend_invocation, backend_request, backend_response_preview, execution_contract,
+        backend_invocation, backend_request, backend_response_preview, execute_live_command_with,
-        execution_preview, execution_stub, plan_command, render_backend_response,
+        execution_contract, execution_preview, execution_stub, plan_command,
-        render_command_summary, render_execution_preview, AuthMode, BackendInvocation,
+        render_backend_response, render_command_summary, render_execution_preview, AuthMode,
-        BackendRequest, BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract,
+        BackendInvocation, BackendRequest, BackendResponsePreview, Cli, CommandPlan, Commands,
-        ExecutionPreview, ExecutionStub,
+        ExecutionContract, ExecutionPreview, ExecutionStub, LiveExecutionResponse,
     };
     use clap::Parser;
     use serde_json::json;
@@ -2240,4 +2261,139 @@ mod tests {
             r#"{"kind":"throughput","model":"qwen-coder","total_tokens":72,"tokens_per_second":42.5}"#
         );
     }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_status_execute_pipeline_uses_transport_and_renders_text() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "--server",
+            "http://127.0.0.1:9999",
+            "--api-key",
+            "secret",
+            "status",
+        ])
+        .unwrap();
+
+        let rendered = execute_live_command_with(&cli, |request| {
+            assert_eq!(
+                request,
+                &BackendRequest::HttpRequest {
+                    method: "GET",
+                    url: "http://127.0.0.1:9999/health".to_owned(),
+                    headers: vec![("authorization".to_owned(), "Bearer secret".to_owned())],
+                    body: None,
+                }
+            );
+
+            LiveExecutionResponse {
+                status_code: 200,
+                body: r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#.to_owned(),
+            }
+        });
+
+        assert_eq!(
+            rendered,
+            "STATUS status=ok model_loaded=true model_id=qwen-coder uptime_secs=12"
+        );
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_models_execute_pipeline_uses_transport_and_renders_json() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "--server",
+            "http://127.0.0.1:9999",
+            "--format",
+            "json",
+            "models",
+            "list",
+        ])
+        .unwrap();
+
+        let rendered = execute_live_command_with(&cli, |request| {
+            assert_eq!(
+                request,
+                &BackendRequest::HttpRequest {
+                    method: "GET",
+                    url: "http://127.0.0.1:9999/v1/models".to_owned(),
+                    headers: vec![],
+                    body: None,
+                }
+            );
+
+            LiveExecutionResponse {
+                status_code: 200,
+                body: r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#
+                    .to_owned(),
+            }
+        });
+
+        assert_eq!(
+            rendered,
+            r#"{"kind":"models","count":2,"ids":["qwen-coder","gemma-3"]}"#
+        );
+    }
+
+    #[test]
+    #[ignore = "autoresearch benchmark"]
+    fn cli_surface_bench_execute_pipeline_uses_transport_and_renders_text() {
+        let cli = Cli::try_parse_from([
+            "localllm",
+            "--server",
+            "http://127.0.0.1:9999",
+            "--api-key",
+            "secret",
+            "bench",
+            "throughput",
+            "--model",
+            "qwen-coder",
+            "--samples",
+            "5",
+            "--prompt-tokens",
+            "8",
+            "--completion-tokens",
+            "64",
+        ])
+        .unwrap();
+
+        let rendered = execute_live_command_with(&cli, |request| {
+            assert_eq!(
+                request,
+                &BackendRequest::HttpRequest {
+                    method: "POST",
+                    url: "http://127.0.0.1:9999/v1/chat/completions".to_owned(),
+                    headers: vec![
+                        ("authorization".to_owned(), "Bearer secret".to_owned()),
+                        ("content-type".to_owned(), "application/json".to_owned()),
+                    ],
+                    body: Some(
+                        json!({
+                            "model": "qwen-coder",
+                            "stream": false,
+                            "temperature": 0.0,
+                            "top_p": 1.0,
+                            "max_completion_tokens": 64,
+                            "messages": [{
+                                "role": "user",
+                                "content": "benchmark token benchmark token benchmark token benchmark token"
+                            }],
+                        })
+                        .to_string(),
+                    ),
+                }
+            );
+
+            LiveExecutionResponse {
+                status_code: 200,
+                body: r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#.to_owned(),
+            }
+        });
+
+        assert_eq!(
+            rendered,
+            "THROUGHPUT model=qwen-coder total_tokens=72 tokens_per_second=42.5"
+        );
+    }
 }

experiments/worklog.md

@@ -334,3 +334,22 @@
 - New objective: raise the CLI benchmark from internal backend response previews to final user-facing live output rendering for the already live-compatible workflows `status`, `models list`, and `bench throughput`.
 - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
 - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
+
+### Run 35: baseline - cli_surface_score=90.476 (KEEP)
+- Timestamp: 2026-03-26 20:06
+- What changed: Expanded the benchmark from backend response previews to final user-facing live output rendering, but left `render_backend_response()` as a shallow `LIVE {summary}` placeholder.
+- Result: cli_surface_score=90.476, passed_cases=57, failed_cases=6, total_cases=63.
+- Insight: The CLI already parsed health, model-list, and throughput responses correctly, but it still lacked final text/json presentation for those live-compatible workflows.
+- Next: render `BackendResponsePreview` into final `STATUS`, `MODELS`, and `THROUGHPUT` text/json outputs.
+
+### Run 36: render cli live outputs - cli_surface_score=100.0 (KEEP)
+- Timestamp: 2026-03-26 20:07
+- What changed: Implemented `render_backend_response()` in `crates/localllm-cli/src/main.rs` so the live-compatible commands now render final text/json output directly from `BackendResponsePreview`, including health summaries, model-id lists, and throughput usage output.
+- Result: cli_surface_score=100.0, passed_cases=63, failed_cases=0, total_cases=63.
+- Insight: The CLI now closes the full loop for the first live-compatible workflows, from parser and planning all the way to final user-facing live output.
+- Next: move from live output rendering to the first real executable handlers for commands that already work today.
+
+## Segment 16: CLI Live Execution
+- New objective: raise the CLI benchmark from final live output rendering to the first real executable handlers for the live-compatible workflows `status`, `models list`, and `bench throughput` using an injectable transport seam.
+- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
+- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.