From 812b486855d156dc890a43e1950103768b169646 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 19:14:00 -0300 Subject: [PATCH] feat: add cli execution contracts --- autoresearch-dashboard.md | 8 +++---- autoresearch.jsonl | 3 +++ autoresearch.md | 1 + crates/localllm-cli/src/main.rs | 41 +++++++++++++++++++++++++++++++-- experiments/worklog.md | 7 ++++++ 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/autoresearch-dashboard.md b/autoresearch-dashboard.md index d9f85ea..728d54b 100644 --- a/autoresearch-dashboard.md +++ b/autoresearch-dashboard.md @@ -1,9 +1,9 @@ -# Autoresearch Dashboard: localllm-cli-execution-plans +# Autoresearch Dashboard: localllm-cli-execution-contracts **Runs:** 1 | **Kept:** 1 | **Discarded:** 0 | **Crashed:** 0 -**Baseline:** cli_surface_score: 69.565% (#19) -**Best:** cli_surface_score: 69.565% (#19, +0.0%) +**Baseline:** cli_surface_score: 76.667% (#21) +**Best:** cli_surface_score: 76.667% (#21, +0.0%) | # | commit | cli_surface_score | status | description | |---|--------|-------------------|--------|-------------| -| 19 | 9154a7b | 69.565% (+0.0%) | keep | baseline | +| 21 | 3d1460b | 76.667% (+0.0%) | keep | baseline | diff --git a/autoresearch.jsonl b/autoresearch.jsonl index f9397cc..1e240eb 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -25,3 +25,6 @@ {"run":18,"commit":"66b30de","metric":100.0,"metrics":{"passed_cases":16,"failed_cases":0,"total_cases":16},"status":"keep","description":"enrich cli semantic summaries","timestamp":1774562434,"segment":6} {"type":"config","name":"localllm-cli-execution-plans","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} {"run":19,"commit":"9154a7b","metric":69.565,"metrics":{"passed_cases":16,"failed_cases":7,"total_cases":23},"status":"keep","description":"baseline","timestamp":1774562688,"segment":7} +{"run":20,"commit":"bc584be","metric":100.0,"metrics":{"passed_cases":23,"failed_cases":0,"total_cases":23},"status":"keep","description":"add cli execution plans","timestamp":1774562759,"segment":7} +{"type":"config","name":"localllm-cli-execution-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":21,"commit":"3d1460b","metric":76.667,"metrics":{"passed_cases":23,"failed_cases":7,"total_cases":30},"status":"keep","description":"baseline","timestamp":1774563087,"segment":8} diff --git a/autoresearch.md b/autoresearch.md index 8efcd3b..b139ef0 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -51,3 +51,4 @@ so later experiments can add real execution behavior without redesigning the UX - Baseline run 19 (`9154a7b`) scored `69.565%`: the CLI still passes all parser and summary tests, but every new plan-oriented workflow fails because `plan_command()` currently returns `SummaryOnly` for all commands. - Run 20 (`bc584be`) introduced concrete `CommandPlan` variants for status, model scan/load, auth enable, config set, throughput bench, and logs, raising segment 7 from `69.565%` to `100.0%` (23/23 passing). - Segment 8 raises the bar from machine-usable plans to execution contracts, checking whether key workflows can already describe the concrete HTTP/local action they intend to perform at the backend/control-plane seam. +- Baseline run 21 (`3d1460b`) scored `76.667%`: the CLI already preserves parser, summary, and plan semantics, but `execution_contract()` still collapses the new workflows to `SummaryOnly { command: "unplanned" }` instead of concrete HTTP/local contracts. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index fff03da..961e4fe 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -331,9 +331,46 @@ fn execution_contract(plan: &CommandPlan) -> ExecutionContract { CommandPlan::SummaryOnly { command } => ExecutionContract::SummaryOnly { command: command.clone(), }, - _ => ExecutionContract::SummaryOnly { - command: "unplanned".to_owned(), + CommandPlan::ProbeStatus { + api_key_present, .. + } => ExecutionContract::HttpGet { + path: "/health".to_owned(), + api_key_present: *api_key_present, }, + CommandPlan::ScanModels { models_dir } => ExecutionContract::LocalModelScan { + models_dir: models_dir.clone(), + }, + CommandPlan::LoadModel { + models_dir, + model, + alias, + } => ExecutionContract::LocalModelLoad { + models_dir: models_dir.clone(), + model: model.clone(), + alias: alias.clone(), + }, + CommandPlan::ConfigureAuth { mode, api_key } => ExecutionContract::LocalAuthConfig { + mode: *mode, + api_key: api_key.clone(), + }, + CommandPlan::SetConfig { key, value } => ExecutionContract::LocalConfigWrite { + key: key.clone(), + value: value.clone(), + }, + CommandPlan::RunThroughputBench { + api_key_present, + samples, + prompt_tokens, + completion_tokens, + .. + } => ExecutionContract::ThroughputRequest { + path: "/v1/chat/completions".to_owned(), + api_key_present: *api_key_present, + samples: *samples, + prompt_tokens: *prompt_tokens, + completion_tokens: *completion_tokens, + }, + CommandPlan::ReadLogs { lines } => ExecutionContract::LocalLogRead { lines: *lines }, } } diff --git a/experiments/worklog.md b/experiments/worklog.md index 5472f94..1f887db 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -188,6 +188,13 @@ - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. +### Run 21: baseline - cli_surface_score=76.667 (KEEP) +- Timestamp: 2026-03-26 19:11 +- What changed: Expanded the benchmark from `CommandPlan` semantics to `ExecutionContract` semantics, requiring the CLI to describe the concrete HTTP or local operation each high-value workflow intends to perform. +- Result: cli_surface_score=76.667, passed_cases=23, failed_cases=7, total_cases=30. +- Insight: The planning layer is now solid, but `execution_contract()` is still stubbed and returns `SummaryOnly { command: "unplanned" }` for every concrete plan, so the CLI cannot yet describe its backend/control-plane seam accurately. +- Next: map the high-value `CommandPlan` variants into concrete HTTP/local execution contracts for status, models scan/load, auth enable, config set, throughput bench, and logs. + ### Run 17: baseline - cli_surface_score=68.75 (KEEP) - Timestamp: 2026-03-26 18:59 - What changed: Raised the benchmark with summary-focused ignored tests that require `render_command_summary()` to preserve output format, models directory, command-specific arguments, and key defaults for status, models load, auth enable, bench throughput, and logs.