chore: expand cli backend response coverage
This commit is contained in:
Evandro Fonseca Junior committed
1 parent
e313c88bdb
commit
bf985ea9ea
7 files changed
+341
-9
No files matched your search
Generated
+130
@@ -20,6 +20,56 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.102"
|
||||
@@ -182,6 +232,52 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
@@ -539,6 +635,12 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
@@ -593,6 +695,16 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "localllm-cli"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
@@ -681,6 +793,12 @@ version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
@@ -919,6 +1037,12 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
@@ -1154,6 +1278,12 @@ version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.22.0"
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# Autoresearch Dashboard: localllm-cli-execution-contracts
|
||||
# Autoresearch Dashboard: localllm-cli-backend-requests
|
||||
|
||||
**Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0
|
||||
**Baseline:** cli_surface_score: 76.667% (#21)
|
||||
**Best:** cli_surface_score: 76.667% (#21, +0.0%)
|
||||
**Runs:** 2 | **Kept:** 2 | **Discarded:** 0 | **Crashed:** 0
|
||||
**Baseline:** cli_surface_score: 94.444% (#31)
|
||||
**Best:** cli_surface_score: 100.0% (#32, +5.9%)
|
||||
|
||||
| # | commit | cli_surface_score | status | description |
|
||||
|---|--------|-------------------|--------|-------------|
|
||||
| 21 | 3d1460b | 76.667% (+0.0%) | keep | baseline |
|
||||
| 31 | f970c05 | 94.444% (+0.0%) | keep | baseline |
|
||||
| 32 | 5de35c5 | 100.0% (+5.9%) | keep | add cli backend request specs |
|
||||
@@ -39,3 +39,9 @@
|
||||
{"run":27,"commit":"7738acf","metric":87.5,"metrics":{"passed_cases":42,"failed_cases":6,"total_cases":48},"status":"keep","description":"baseline","timestamp":1774564340,"segment":11}
|
||||
{"run":28,"commit":"99e5041","metric":100.0,"metrics":{"passed_cases":48,"failed_cases":0,"total_cases":48},"status":"keep","description":"render cli dry-run previews","timestamp":1774564465,"segment":11}
|
||||
{"type":"config","name":"localllm-cli-backend-invocations","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
||||
{"run":29,"commit":"1ff5ba3","metric":94.118,"metrics":{"passed_cases":48,"failed_cases":3,"total_cases":51},"status":"keep","description":"baseline","timestamp":1774564791,"segment":12}
|
||||
{"run":30,"commit":"bf40b2f","metric":100.0,"metrics":{"passed_cases":51,"failed_cases":0,"total_cases":51},"status":"keep","description":"add cli backend invocation specs","timestamp":1774564898,"segment":12}
|
||||
{"type":"config","name":"localllm-cli-backend-requests","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
||||
{"run":31,"commit":"f970c05","metric":94.444,"metrics":{"passed_cases":51,"failed_cases":3,"total_cases":54},"status":"keep","description":"baseline","timestamp":1774565247,"segment":13}
|
||||
{"run":32,"commit":"5de35c5","metric":100.0,"metrics":{"passed_cases":54,"failed_cases":0,"total_cases":54},"status":"keep","description":"add cli backend request specs","timestamp":1774565368,"segment":13}
|
||||
{"type":"config","name":"localllm-cli-backend-responses","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
||||
@@ -63,3 +63,9 @@ so later experiments can add real execution behavior without redesigning the UX
|
||||
- Baseline run 27 (`7738acf`) scored `87.5%`: the CLI already had parser/summary/plan/contract/stub/preview layers, but `render_execution_preview()` still collapsed everything to `DRY RUN {summary}` instead of final text/json dry-run output.
|
||||
- Run 28 (`99e5041`) rendered the highest-value `ExecutionPreview` values into final user-facing text/json dry-run output, raising segment 11 from `87.5%` to `100.0%` (48/48 passing).
|
||||
- Segment 12 raises the bar from dry-run UX to the first real backend-call specifications for commands that can already work today, specifically `status`, `models list`, and `bench throughput`.
|
||||
- Baseline run 29 (`1ff5ba3`) scored `94.118%`: every earlier CLI layer still passed, but `backend_invocation()` was still a placeholder and the new backend-facing tests for live status/models/bench requests failed.
|
||||
- Run 30 (`bf40b2f`) added concrete backend invocation specs for `GET /health`, `GET /v1/models`, and `POST /v1/chat/completions` with preserved `--model`, token budget, server base URL, and auth presence, raising segment 12 from `94.118%` to `100.0%` (51/51 passing).
|
||||
- Segment 13 raises the bar again from backend invocation specs to executable backend request construction, checking whether the same live-compatible commands can now produce final HTTP method/url/header/body request objects with deterministic JSON payloads.
|
||||
- Run 31 (`f970c05`) scored `94.444%` because `backend_request()` was still a placeholder and the new backend-request tests failed for `status`, `models list`, and `bench throughput`.
|
||||
- Run 32 (`5de35c5`) introduced concrete executable backend request specs with bearer/content-type headers and deterministic JSON benchmark payloads, raising segment 13 from `94.444%` to `100.0%` (54/54 passing).
|
||||
- Segment 14 raises the bar from executable backend request specs to backend response interpretation, checking whether the same live-compatible commands can parse real response bodies into CLI-facing previews for health, model lists, and throughput usage metrics.
|
||||
@@ -12,4 +12,5 @@ path = "src/main.rs"
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -1,4 +1,5 @@
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
enum OutputFormat {
|
||||
@@ -321,6 +322,57 @@ enum BackendRequest {
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum BackendResponsePreview {
|
||||
SummaryOnly {
|
||||
command: String,
|
||||
},
|
||||
Health {
|
||||
status: String,
|
||||
model_loaded: bool,
|
||||
model_id: Option<String>,
|
||||
uptime_secs: u64,
|
||||
},
|
||||
Models {
|
||||
count: usize,
|
||||
ids: Vec<String>,
|
||||
},
|
||||
Throughput {
|
||||
model: String,
|
||||
total_tokens: u32,
|
||||
tokens_per_second: f64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CliHealthResponse {
|
||||
status: String,
|
||||
model_loaded: bool,
|
||||
model_id: Option<String>,
|
||||
uptime_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CliModelList {
|
||||
data: Vec<CliModelInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CliModelInfo {
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CliBenchResponse {
|
||||
usage: CliUsage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CliUsage {
|
||||
total_tokens: u32,
|
||||
tokens_per_second: f64,
|
||||
}
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
if cli.dry_run {
|
||||
@@ -695,6 +747,33 @@ fn backend_request(cli: &Cli) -> BackendRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn backend_response_preview(cli: &Cli, _status_code: u16, _body: &str) -> BackendResponsePreview {
|
||||
match &cli.command {
|
||||
Commands::Status => BackendResponsePreview::SummaryOnly {
|
||||
command: "response-unimplemented".to_owned(),
|
||||
},
|
||||
Commands::Models(models) => match &models.command {
|
||||
ModelsSubcommand::List => BackendResponsePreview::SummaryOnly {
|
||||
command: "response-unimplemented".to_owned(),
|
||||
},
|
||||
_ => BackendResponsePreview::SummaryOnly {
|
||||
command: command_summary(&cli.command),
|
||||
},
|
||||
},
|
||||
Commands::Bench(bench) => match &bench.command {
|
||||
BenchSubcommand::Throughput(_) => BackendResponsePreview::SummaryOnly {
|
||||
command: "response-unimplemented".to_owned(),
|
||||
},
|
||||
_ => BackendResponsePreview::SummaryOnly {
|
||||
command: command_summary(&cli.command),
|
||||
},
|
||||
},
|
||||
_ => BackendResponsePreview::SummaryOnly {
|
||||
command: command_summary(&cli.command),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn bearer_headers(cli: &Cli) -> Vec<(String, String)> {
|
||||
cli.api_key
|
||||
.as_ref()
|
||||
@@ -822,10 +901,11 @@ fn bench_command_summary(command: &BenchCommand) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
backend_invocation, backend_request, execution_contract, execution_preview, execution_stub,
|
||||
plan_command, render_command_summary, render_execution_preview, AuthMode,
|
||||
BackendInvocation, BackendRequest, Cli, CommandPlan, Commands, ExecutionContract,
|
||||
ExecutionPreview, ExecutionStub,
|
||||
backend_invocation, backend_request, backend_response_preview, execution_contract,
|
||||
execution_preview, execution_stub, plan_command, render_command_summary,
|
||||
render_execution_preview, AuthMode, BackendInvocation, BackendRequest,
|
||||
BackendResponsePreview, Cli, CommandPlan, Commands, ExecutionContract, ExecutionPreview,
|
||||
ExecutionStub,
|
||||
};
|
||||
use clap::Parser;
|
||||
use serde_json::json;
|
||||
@@ -1874,4 +1954,74 @@ mod tests {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "autoresearch benchmark"]
|
||||
fn cli_surface_status_response_parses_health_payload() {
|
||||
let cli = Cli::try_parse_from(["localllm", "status"]).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
backend_response_preview(
|
||||
&cli,
|
||||
200,
|
||||
r#"{"status":"ok","model_loaded":true,"model_id":"qwen-coder","uptime_secs":12}"#,
|
||||
),
|
||||
BackendResponsePreview::Health {
|
||||
status: "ok".to_owned(),
|
||||
model_loaded: true,
|
||||
model_id: Some("qwen-coder".to_owned()),
|
||||
uptime_secs: 12,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "autoresearch benchmark"]
|
||||
fn cli_surface_models_list_response_collects_ids() {
|
||||
let cli = Cli::try_parse_from(["localllm", "models", "list"]).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
backend_response_preview(
|
||||
&cli,
|
||||
200,
|
||||
r#"{"object":"list","data":[{"id":"qwen-coder"},{"id":"gemma-3"}]}"#,
|
||||
),
|
||||
BackendResponsePreview::Models {
|
||||
count: 2,
|
||||
ids: vec!["qwen-coder".to_owned(), "gemma-3".to_owned()],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "autoresearch benchmark"]
|
||||
fn cli_surface_bench_response_reads_usage_metrics() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"localllm",
|
||||
"bench",
|
||||
"throughput",
|
||||
"--model",
|
||||
"qwen-coder",
|
||||
"--samples",
|
||||
"5",
|
||||
"--prompt-tokens",
|
||||
"8",
|
||||
"--completion-tokens",
|
||||
"64",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
backend_response_preview(
|
||||
&cli,
|
||||
200,
|
||||
r#"{"usage":{"prompt_tokens":8,"completion_tokens":64,"total_tokens":72,"tokens_per_second":42.5}}"#,
|
||||
),
|
||||
BackendResponsePreview::Throughput {
|
||||
model: "qwen-coder".to_owned(),
|
||||
total_tokens: 72,
|
||||
tokens_per_second: 42.5,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -277,3 +277,41 @@
|
||||
- New objective: raise the CLI benchmark from dry-run rendering to the first real backend-call specifications for commands that can already work today, especially `status`, `models list`, and `bench throughput`.
|
||||
- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
|
||||
- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
|
||||
|
||||
## Segment 13: CLI Backend Requests
|
||||
- New objective: raise the CLI benchmark from backend invocation specs to executable backend request construction for the commands that already align with live server endpoints today.
|
||||
- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
|
||||
- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
|
||||
|
||||
### Run 29: baseline - cli_surface_score=94.118 (KEEP)
|
||||
- Timestamp: 2026-03-26 19:39
|
||||
- What changed: Expanded the benchmark from final dry-run rendering to backend invocation specs for commands that can already work today, adding ignored tests for live `GET /health`, live `GET /v1/models`, and a throughput `POST /v1/chat/completions` request that preserves the selected model and token budget.
|
||||
- Result: cli_surface_score=94.118, passed_cases=48, failed_cases=3, total_cases=51.
|
||||
- Insight: The CLI already handled every previous parser/planning/dry-run layer, but `backend_invocation()` was still a placeholder, and `models list` had not yet been promoted into the real backend-facing seam.
|
||||
- Next: implement concrete backend invocation specs for `status`, `models list`, and `bench throughput` using the resolved server base URL and auth presence.
|
||||
|
||||
### Run 30: add cli backend invocation specs - cli_surface_score=100.0 (KEEP)
|
||||
- Timestamp: 2026-03-26 19:41
|
||||
- What changed: Implemented `backend_invocation(&Cli)` in `crates/localllm-cli/src/main.rs` so `status` maps to live `GET /health`, `models list` maps to live `GET /v1/models`, and `bench throughput` maps to a live `POST /v1/chat/completions` invocation that preserves `--model`, samples, token budget, and auth presence.
|
||||
- Result: cli_surface_score=100.0, passed_cases=51, failed_cases=0, total_cases=51.
|
||||
- Insight: The CLI now crosses from dry-run-only UX into concrete backend-facing invocation specs for commands that can already work against the current server today, without needing the future control-plane endpoints yet.
|
||||
- Next: raise the bar from backend invocation specs to actual request payload construction or the first executable handlers for these already-supported commands.
|
||||
|
||||
### Run 31: baseline - cli_surface_score=94.444 (KEEP)
|
||||
- Timestamp: 2026-03-26 19:47
|
||||
- What changed: Expanded the benchmark from backend invocation specs to executable backend request construction, requiring final HTTP method/url/header/body request objects with deterministic JSON payloads for `status`, `models list`, and `bench throughput`.
|
||||
- Result: cli_surface_score=94.444, passed_cases=51, failed_cases=3, total_cases=54.
|
||||
- Insight: The CLI already had parser, summaries, plans, contracts, stubs, previews, dry-run rendering, and invocation specs, but `backend_request(&Cli)` still returned `SummaryOnly { command: "request-unimplemented" }` instead of final request objects.
|
||||
- Next: map `BackendInvocation::HttpCall` into executable `BackendRequest::HttpRequest` values with bearer headers, JSON content-type, and deterministic benchmark payload generation.
|
||||
|
||||
### Run 32: add cli backend request specs - cli_surface_score=100.0 (KEEP)
|
||||
- Timestamp: 2026-03-26 19:49
|
||||
- What changed: Implemented `backend_request(&Cli)` in `crates/localllm-cli/src/main.rs` plus helpers for bearer headers, JSON headers, deterministic benchmark prompt construction, and exact chat payload generation for throughput requests.
|
||||
- Result: cli_surface_score=100.0, passed_cases=54, failed_cases=0, total_cases=54.
|
||||
- Insight: The first live-compatible CLI workflows now preserve the full executable backend request boundary - method, URL, auth headers, content-type, and deterministic JSON body - while still remaining execution-free.
|
||||
- Next: raise the bar from request construction to the first live response-handling or executable handler seams for the same workflows.
|
||||
|
||||
## Segment 14: CLI Backend Responses
|
||||
- New objective: raise the CLI benchmark from executable backend request construction to backend response interpretation for the same live-compatible workflows, starting with `status`, `models list`, and `bench throughput`.
|
||||
- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`.
|
||||
- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.
|
||||
Reference in new issue
Block a user