diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 46df334..0dddbef 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -48,3 +48,6 @@ {"run":33,"commit":"8cf03e3","metric":94.737,"metrics":{"passed_cases":54,"failed_cases":3,"total_cases":57},"status":"keep","description":"baseline","timestamp":1774566069,"segment":14} {"run":34,"commit":"0506a4c","metric":100.0,"metrics":{"passed_cases":57,"failed_cases":0,"total_cases":57},"status":"keep","description":"add cli backend response previews","timestamp":1774566132,"segment":14} {"type":"config","name":"localllm-cli-live-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":35,"commit":"e1ada2c","metric":90.476,"metrics":{"passed_cases":57,"failed_cases":6,"total_cases":63},"status":"keep","description":"baseline","timestamp":1774566405,"segment":15} +{"run":36,"commit":"c54107d","metric":100.0,"metrics":{"passed_cases":63,"failed_cases":0,"total_cases":63},"status":"keep","description":"render cli live outputs","timestamp":1774566456,"segment":15} +{"type":"config","name":"localllm-cli-live-execution","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index d8b33ba..c71a643 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -72,3 +72,6 @@ so later experiments can add real execution behavior without redesigning the UX - Run 33 (`8cf03e3`) scored `94.737%` because `backend_response_preview()` was still a placeholder and the new response-parsing tests failed for `status`, `models list`, and `bench throughput`. - Run 34 (`0506a4c`) introduced CLI-local backend response previews for health payloads, model list ids/count, and throughput usage metrics, raising segment 14 from `94.737%` to `100.0%` (57/57 passing). - Segment 15 raises the bar from internal backend response previews to final user-facing live output rendering, checking whether the same live-compatible commands can render text/json output directly from parsed backend responses. +- Run 35 (`e1ada2c`) scored `90.476%` because `render_backend_response()` still collapsed the live-compatible workflows to `LIVE {summary}` instead of final text/json output. +- Run 36 (`c54107d`) rendered the first live-compatible backend responses into final user-facing text/json output, raising segment 15 from `90.476%` to `100.0%` (63/63 passing). +- Segment 16 raises the bar from live output rendering to the first real executable handlers, checking whether the same live-compatible workflows can execute through an injectable transport seam and return the final rendered output end-to-end. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index ab8772e..8398a01 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -344,6 +344,12 @@ enum BackendResponsePreview { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct LiveExecutionResponse { + status_code: u16, + body: String, +} + #[derive(Debug, Deserialize)] struct CliHealthResponse { status: String, @@ -518,6 +524,21 @@ fn render_backend_response(cli: &Cli, status_code: u16, body: &str) -> String { } } +fn execute_live_command_with(cli: &Cli, mut transport: F) -> String +where + F: FnMut(&BackendRequest) -> LiveExecutionResponse, +{ + let request = backend_request(cli); + + match request { + BackendRequest::SummaryOnly { command } => format!("LIVE {}", command), + _ => { + let _response = transport(&request); + format!("LIVE EXEC {}", render_command_summary(cli)) + } + } +} + fn render_json_optional_string(value: Option<&str>) -> String { match value { Some(value) => format!(r#""{}""#, value), @@ -999,11 +1020,11 @@ fn bench_command_summary(command: &BenchCommand) -> String { #[cfg(test)] mod tests { use super::{ - backend_invocation, backend_request, backend_response_preview, execution_contract, - execution_preview, execution_stub, plan_command, render_backend_response, - render_command_summary, render_execution_preview, AuthMode, BackendInvocation, - BackendRequest, BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, - ExecutionPreview, ExecutionStub, + backend_invocation, backend_request, backend_response_preview, execute_live_command_with, + execution_contract, execution_preview, execution_stub, plan_command, + render_backend_response, render_command_summary, render_execution_preview, AuthMode, + BackendInvocation, BackendRequest, BackendResponsePreview, Cli, CommandPlan, Commands, + ExecutionContract, ExecutionPreview, ExecutionStub, LiveExecutionResponse, }; use clap::Parser; use serde_json::json; @@ -2240,4 +2261,139 @@ mod tests { r#"{"kind":"throughput","model":"qwen-coder","total_tokens":72,"tokens_per_second":42.5}"# ); } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_status_execute_pipeline_uses_transport_and_renders_text() { + let cli = Cli::try_parse_from([ + "localllm", + "--server", + "http://127.0.0.1:9999", + "--api-key", + "secret", + "status", + ]) + .unwrap(); + + let rendered = execute_live_command_with(&cli, |request| { + assert_eq!( + request, + &BackendRequest::HttpRequest { + method: "GET", + url: "http://127.0.0.1:9999/health".to_owned(), + headers: vec![("authorization".to_owned(), "Bearer secret".to_owned())], + body: None, + } + ); + + LiveExecutionResponse { + status_code: 200, + body: r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#.to_owned(), + } + }); + + assert_eq!( + rendered, + "STATUS status=ok model_loaded=true model_id=qwen-coder uptime_secs=12" + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_models_execute_pipeline_uses_transport_and_renders_json() { + let cli = Cli::try_parse_from([ + "localllm", + "--server", + "http://127.0.0.1:9999", + "--format", + "json", + "models", + "list", + ]) + .unwrap(); + + let rendered = execute_live_command_with(&cli, |request| { + assert_eq!( + request, + &BackendRequest::HttpRequest { + method: "GET", + url: "http://127.0.0.1:9999/v1/models".to_owned(), + headers: vec![], + body: None, + } + ); + + LiveExecutionResponse { + status_code: 200, + body: r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"# + .to_owned(), + } + }); + + assert_eq!( + rendered, + r#"{"kind":"models","count":2,"ids":["qwen-coder","gemma-3"]}"# + ); + } + + #[test] + #[ignore = "autoresearch benchmark"] + fn cli_surface_bench_execute_pipeline_uses_transport_and_renders_text() { + let cli = Cli::try_parse_from([ + "localllm", + "--server", + "http://127.0.0.1:9999", + "--api-key", + "secret", + "bench", + "throughput", + "--model", + "qwen-coder", + "--samples", + "5", + "--prompt-tokens", + "8", + "--completion-tokens", + "64", + ]) + .unwrap(); + + let rendered = execute_live_command_with(&cli, |request| { + assert_eq!( + request, + &BackendRequest::HttpRequest { + method: "POST", + url: "http://127.0.0.1:9999/v1/chat/completions".to_owned(), + headers: vec![ + ("authorization".to_owned(), "Bearer secret".to_owned()), + ("content-type".to_owned(), "application/json".to_owned()), + ], + body: Some( + json!({ + "model": "qwen-coder", + "stream": false, + "temperature": 0.0, + "top_p": 1.0, + "max_completion_tokens": 64, + "messages": [{ + "role": "user", + "content": "benchmark token benchmark token benchmark token benchmark token" + }], + }) + .to_string(), + ), + } + ); + + LiveExecutionResponse { + status_code: 200, + body: r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#.to_owned(), + } + }); + + assert_eq!( + rendered, + "THROUGHPUT model=qwen-coder total_tokens=72 tokens_per_second=42.5" + ); + } } diff --git a/experiments/worklog.md b/experiments/worklog.md index e911ff2..71a41cd 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -334,3 +334,22 @@ - New objective: raise the CLI benchmark from internal backend response previews to final user-facing live output rendering for the already live-compatible workflows `status`, `models list`, and `bench throughput`. - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 35: baseline - cli_surface_score=90.476 (KEEP) +- Timestamp: 2026-03-26 20:06 +- What changed: Expanded the benchmark from backend response previews to final user-facing live output rendering, but left `render_backend_response()` as a shallow `LIVE {summary}` placeholder. +- Result: cli_surface_score=90.476, passed_cases=57, failed_cases=6, total_cases=63. +- Insight: The CLI already parsed health, model-list, and throughput responses correctly, but it still lacked final text/json presentation for those live-compatible workflows. +- Next: render `BackendResponsePreview` into final `STATUS`, `MODELS`, and `THROUGHPUT` text/json outputs. + +### Run 36: render cli live outputs - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 20:07 +- What changed: Implemented `render_backend_response()` in `crates/localllm-cli/src/main.rs` so the live-compatible commands now render final text/json output directly from `BackendResponsePreview`, including health summaries, model-id lists, and throughput usage output. +- Result: cli_surface_score=100.0, passed_cases=63, failed_cases=0, total_cases=63. +- Insight: The CLI now closes the full loop for the first live-compatible workflows, from parser and planning all the way to final user-facing live output. +- Next: move from live output rendering to the first real executable handlers for commands that already work today. + +## Segment 16: CLI Live Execution +- New objective: raise the CLI benchmark from final live output rendering to the first real executable handlers for the live-compatible workflows `status`, `models list`, and `bench throughput` using an injectable transport seam. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.