From 9435b208acb5cb992b9818fc79228167ccfcd4ef Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 20:06:45 -0300 Subject: [PATCH] chore: expand cli live output coverage --- autoresearch.jsonl | 3 + autoresearch.md | 3 + crates/localllm-cli/src/main.rs | 130 +++++++++++++++++++++++++++++++- experiments/worklog.md | 19 +++++ 4 files changed, 151 insertions(+), 4 deletions(-) diff --git a/autoresearch.jsonl b/autoresearch.jsonl index e508a1e..46df334 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -45,3 +45,6 @@ {"run":31,"commit":"f970c05","metric":94.444,"metrics":{"passed_cases":51,"failed_cases":3,"total_cases":54},"status":"keep","description":"baseline","timestamp":1774565247,"segment":13} {"run":32,"commit":"5de35c5","metric":100.0,"metrics":{"passed_cases":54,"failed_cases":0,"total_cases":54},"status":"keep","description":"add cli backend request specs","timestamp":1774565368,"segment":13} {"type":"config","name":"localllm-cli-backend-responses","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":33,"commit":"8cf03e3","metric":94.737,"metrics":{"passed_cases":54,"failed_cases":3,"total_cases":57},"status":"keep","description":"baseline","timestamp":1774566069,"segment":14} +{"run":34,"commit":"0506a4c","metric":100.0,"metrics":{"passed_cases":57,"failed_cases":0,"total_cases":57},"status":"keep","description":"add cli backend response previews","timestamp":1774566132,"segment":14} +{"type":"config","name":"localllm-cli-live-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index 72fb729..d8b33ba 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -69,3 +69,6 @@ so later experiments can add real execution behavior without redesigning the UX - Run 31 (`f970c05`) scored `94.444%` because `backend_request()` was still a placeholder and the new backend-request tests failed for `status`, `models list`, and `bench throughput`. - Run 32 (`5de35c5`) introduced concrete executable backend request specs with bearer/content-type headers and deterministic JSON benchmark payloads, raising segment 13 from `94.444%` to `100.0%` (54/54 passing). - Segment 14 raises the bar from executable backend request specs to backend response interpretation, checking whether the same live-compatible commands can parse real response bodies into CLI-facing previews for health, model lists, and throughput usage metrics. +- Run 33 (`8cf03e3`) scored `94.737%` because `backend_response_preview()` was still a placeholder and the new response-parsing tests failed for `status`, `models list`, and `bench throughput`. +- Run 34 (`0506a4c`) introduced CLI-local backend response previews for health payloads, model list ids/count, and throughput usage metrics, raising segment 14 from `94.737%` to `100.0%` (57/57 passing). +- Segment 15 raises the bar from internal backend response previews to final user-facing live output rendering, checking whether the same live-compatible commands can render text/json output directly from parsed backend responses. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index 599262f..a8cec6e 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -445,6 +445,10 @@ fn render_execution_preview(cli: &Cli) -> String { } } +fn render_backend_response(cli: &Cli, _status_code: u16, _body: &str) -> String { + format!("LIVE {}", render_command_summary(cli)) +} + fn render_json_body(body: Option<&str>) -> String { match body { Some(body) => format!(r#""{}""#, body), @@ -920,10 +924,10 @@ fn bench_command_summary(command: &BenchCommand) -> String { mod tests { use super::{ backend_invocation, backend_request, backend_response_preview, execution_contract, - execution_preview, execution_stub, plan_command, render_command_summary, - render_execution_preview, AuthMode, BackendInvocation, BackendRequest, - BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, ExecutionPreview, - ExecutionStub, + execution_preview, execution_stub, plan_command, render_backend_response, + render_command_summary, render_execution_preview, AuthMode, BackendInvocation, + BackendRequest, BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, + ExecutionPreview, ExecutionStub, }; use clap::Parser; use serde_json::json; @@ -2042,4 +2046,122 @@ mod tests { } ); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_live_text_renders_health_summary() { + let cli = Cli::try_parse_from(["localllm", "status"]).unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#, + ), + "STATUS status=ok model_loaded=true model_id=qwen-coder uptime_secs=12" + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_live_json_renders_health_payload() { + let cli = Cli::try_parse_from(["localllm", "--format", "json", "status"]).unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#, + ), + r#"{"kind":"health","status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"# + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_live_text_renders_model_ids() { + let cli = Cli::try_parse_from(["localllm", "models", "list"]).unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#, + ), + "MODELS count=2 ids=qwen-coder,gemma-3" + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_live_json_renders_model_ids() { + let cli = Cli::try_parse_from(["localllm", "--format", "json", "models", "list"]).unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#, + ), + r#"{"kind":"models","count":2,"ids":["qwen-coder","gemma-3"]}"# + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_live_text_renders_usage_summary() { + let cli = Cli::try_parse_from([ + "localllm", + "bench", + "throughput", + "--model", + "qwen-coder", + "--samples", + "5", + "--prompt-tokens", + "8", + "--completion-tokens", + "64", + ]) + .unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#, + ), + "THROUGHPUT model=qwen-coder total_tokens=72 tokens_per_second=42.5" + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_live_json_renders_usage_summary() { + let cli = Cli::try_parse_from([ + "localllm", + "--format", + "json", + "bench", + "throughput", + "--model", + "qwen-coder", + "--samples", + "5", + "--prompt-tokens", + "8", + "--completion-tokens", + "64", + ]) + .unwrap(); + + assert_eq!( + render_backend_response( + &cli, + 200, + r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#, + ), + r#"{"kind":"throughput","model":"qwen-coder","total_tokens":72,"tokens_per_second":42.5}"# + ); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index d43c87a..e911ff2 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -315,3 +315,22 @@ - New objective: raise the CLI benchmark from executable backend request construction to backend response interpretation for the same live-compatible workflows, starting with `status`, `models list`, and `bench throughput`. - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 33: baseline - cli_surface_score=94.737 (KEEP) +- Timestamp: 2026-03-26 20:01 +- What changed: Added the segment 14 response-preview seam and local deserializable DTOs for health, models list, and throughput usage responses, but left `backend_response_preview()` intentionally unimplemented for the new live-compatible workflows. +- Result: cli_surface_score=94.737, passed_cases=54, failed_cases=3, total_cases=57. +- Insight: The CLI already had every layer through executable backend request construction, but it still collapsed real response parsing for `status`, `models list`, and `bench throughput` to `SummaryOnly { command: "response-unimplemented" }`. +- Next: parse the real response bodies into CLI-local previews for health status, model ids/count, and throughput usage metrics. + +### Run 34: add cli backend response previews - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 20:02 +- What changed: Implemented `backend_response_preview()` in `crates/localllm-cli/src/main.rs` so `status` parses health payloads, `models list` collects model ids/count, and `bench throughput` reads `usage.total_tokens` plus `usage.tokens_per_second` into CLI-local preview values. +- Result: cli_surface_score=100.0, passed_cases=57, failed_cases=0, total_cases=57. +- Insight: The first live-compatible CLI workflows now preserve the full response side of the contract too: request construction plus deterministic response interpretation without forcing reuse of server-only DTOs. +- Next: raise the bar from response previews to the first real executable handlers or user-facing live output rendering for commands that already work against the current backend. + +## Segment 15: CLI Live Output +- New objective: raise the CLI benchmark from internal backend response previews to final user-facing live output rendering for the already live-compatible workflows `status`, `models list`, and `bench throughput`. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.