From f538fc60bbc2f6e1fad196840e30aeb85ad431c4 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 26 Mar 2026 19:41:38 -0300 Subject: [PATCH] feat: add cli backend invocation specs --- autoresearch.jsonl | 9 ++++++ autoresearch.md | 9 ++++++ crates/localllm-cli/src/main.rs | 41 +++++++++++++++++++----- experiments/worklog.md | 57 +++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/autoresearch.jsonl b/autoresearch.jsonl index caa81e2..9f3324b 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -30,3 +30,12 @@ {"run":21,"commit":"3d1460b","metric":76.667,"metrics":{"passed_cases":23,"failed_cases":7,"total_cases":30},"status":"keep","description":"baseline","timestamp":1774563087,"segment":8} {"run":22,"commit":"38acb52","metric":100.0,"metrics":{"passed_cases":30,"failed_cases":0,"total_cases":30},"status":"keep","description":"add cli execution contracts","timestamp":1774563260,"segment":8} {"type":"config","name":"localllm-cli-execution-stubs","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":23,"commit":"7030290","metric":83.333,"metrics":{"passed_cases":30,"failed_cases":6,"total_cases":36},"status":"keep","description":"baseline","timestamp":1774563600,"segment":9} +{"run":24,"commit":"50d435a","metric":100.0,"metrics":{"passed_cases":36,"failed_cases":0,"total_cases":36},"status":"keep","description":"add cli execution stubs","timestamp":1774563741,"segment":9} +{"type":"config","name":"localllm-cli-execution-previews","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":25,"commit":"cd480c4","metric":85.714,"metrics":{"passed_cases":36,"failed_cases":6,"total_cases":42},"status":"keep","description":"baseline","timestamp":1774564038,"segment":10} +{"run":26,"commit":"c8ef7cf","metric":100.0,"metrics":{"passed_cases":42,"failed_cases":0,"total_cases":42},"status":"keep","description":"add cli execution previews","timestamp":1774564103,"segment":10} +{"type":"config","name":"localllm-cli-dry-run-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} +{"run":27,"commit":"7738acf","metric":87.5,"metrics":{"passed_cases":42,"failed_cases":6,"total_cases":48},"status":"keep","description":"baseline","timestamp":1774564340,"segment":11} +{"run":28,"commit":"99e5041","metric":100.0,"metrics":{"passed_cases":48,"failed_cases":0,"total_cases":48},"status":"keep","description":"render cli dry-run previews","timestamp":1774564465,"segment":11} +{"type":"config","name":"localllm-cli-backend-invocations","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"} diff --git a/autoresearch.md b/autoresearch.md index bd0e75f..9e476d0 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -54,3 +54,12 @@ so later experiments can add real execution behavior without redesigning the UX - Baseline run 21 (`3d1460b`) scored `76.667%`: the CLI already preserves parser, summary, and plan semantics, but `execution_contract()` still collapses the new workflows to `SummaryOnly { command: "unplanned" }` instead of concrete HTTP/local contracts. - Run 22 (`38acb52`) mapped the highest-value `CommandPlan` variants into concrete `ExecutionContract` values for `/health`, `/v1/chat/completions`, local model scan/load, local auth/config mutation, and local log reads, raising segment 8 from `76.667%` to `100.0%` (30/30 passing). - Segment 9 raises the bar from contract shape to execution stubs, checking whether the same high-value workflows can already describe concrete HTTP method/path/body details or named local actions that a future real executor can run directly. +- Baseline run 23 (`7030290`) scored `83.333%`: parser, summaries, plans, and execution contracts still pass, but `execution_stub()` is only a placeholder and returns `SummaryOnly { command: "unimplemented" }` instead of concrete HTTP/local stubs for the highest-value workflows. +- Run 24 (`50d435a`) mapped the highest-value `ExecutionContract` variants into concrete `ExecutionStub` values for `GET /health`, `POST /v1/chat/completions`, local model scan/load, local auth config, and local log tail actions, raising segment 9 from `83.333%` to `100.0%` (36/36 passing). +- Segment 10 raises the bar from execution stubs to dry-run previews, checking whether the same high-value workflows can expand into full previewable HTTP URLs or local-effect descriptions that are ready to show in a future `--dry-run` UX. +- Baseline run 25 (`cd480c4`) scored `85.714%`: parser, summaries, plans, contracts, and stubs still pass, but `execution_preview()` remained a placeholder and returned `SummaryOnly { command: "preview-unimplemented" }` instead of final dry-run previews. +- Run 26 (`c8ef7cf`) mapped the highest-value execution stubs into concrete `ExecutionPreview` values with full health/chat URLs and readable local-effect descriptions, raising segment 10 from `85.714%` to `100.0%` (42/42 passing). +- Segment 11 raises the bar from preview objects to final `--dry-run` output, checking whether the CLI can now render those previews as actual text/json UX for the same high-value workflows. +- Baseline run 27 (`7738acf`) scored `87.5%`: the CLI already had parser/summary/plan/contract/stub/preview layers, but `render_execution_preview()` still collapsed everything to `DRY RUN {summary}` instead of final text/json dry-run output. +- Run 28 (`99e5041`) rendered the highest-value `ExecutionPreview` values into final user-facing text/json dry-run output, raising segment 11 from `87.5%` to `100.0%` (48/48 passing). +- Segment 12 raises the bar from dry-run UX to the first real backend-call specifications for commands that can already work today, specifically `status`, `models list`, and `bench throughput`. diff --git a/crates/localllm-cli/src/main.rs b/crates/localllm-cli/src/main.rs index b9ce4bc..e4a37c4 100644 --- a/crates/localllm-cli/src/main.rs +++ b/crates/localllm-cli/src/main.rs @@ -614,15 +614,40 @@ fn preview_effect(action: &str, detail: Option<&str>) -> String { } fn backend_invocation(cli: &Cli) -> BackendInvocation { - let preview = execution_preview( - cli, - &execution_stub(&execution_contract(&plan_command(cli))), - ); - - match preview { - ExecutionPreview::SummaryOnly { command } => BackendInvocation::SummaryOnly { command }, + match &cli.command { + Commands::Status => BackendInvocation::HttpCall { + method: "GET", + url: format!("{}/health", cli.server.trim_end_matches('/')), + api_key_present: cli.api_key.is_some(), + body: None, + }, + Commands::Models(models) => match &models.command { + ModelsSubcommand::List => BackendInvocation::HttpCall { + method: "GET", + url: format!("{}/v1/models", cli.server.trim_end_matches('/')), + api_key_present: cli.api_key.is_some(), + body: None, + }, + _ => BackendInvocation::SummaryOnly { + command: command_summary(&cli.command), + }, + }, + Commands::Bench(bench) => match &bench.command { + BenchSubcommand::Throughput(args) => BackendInvocation::HttpCall { + method: "POST", + url: format!("{}/v1/chat/completions", cli.server.trim_end_matches('/')), + api_key_present: cli.api_key.is_some(), + body: Some(format!( + "model={} samples={} prompt_tokens={} completion_tokens={} stream=false", + args.model, args.samples, args.prompt_tokens, args.completion_tokens + )), + }, + _ => BackendInvocation::SummaryOnly { + command: command_summary(&cli.command), + }, + }, _ => BackendInvocation::SummaryOnly { - command: "backend-unimplemented".to_owned(), + command: command_summary(&cli.command), }, } } diff --git a/experiments/worklog.md b/experiments/worklog.md index 1feb396..68fa4ce 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -220,3 +220,60 @@ - New objective: raise the CLI benchmark from abstract execution contracts to concrete execution stubs that preserve HTTP method/path/body semantics or named local actions for the highest-value workflows. - Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. - Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 23: baseline - cli_surface_score=83.333 (KEEP) +- Timestamp: 2026-03-26 19:20 +- What changed: Expanded the benchmark from execution contracts to concrete execution stubs, adding ignored tests for HTTP method/path/body semantics and named local actions for status, model scan/load, auth enable, throughput bench, and logs. +- Result: cli_surface_score=83.333, passed_cases=30, failed_cases=6, total_cases=36. +- Insight: The CLI already knows how to parse commands and describe execution contracts, but `execution_stub()` is still just a placeholder returning `SummaryOnly { command: "unimplemented" }` for every high-value workflow. +- Next: implement concrete stub mapping so status and throughput produce execution-ready HTTP request shapes and local workflows emit named local actions with preserved detail. + +### Run 24: add cli execution stubs - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:22 +- What changed: Implemented concrete `execution_stub()` mapping in `crates/localllm-cli/src/main.rs`, turning the highest-value execution contracts into execution-ready HTTP requests and named local actions with preserved detail. +- Result: cli_surface_score=100.0, passed_cases=36, failed_cases=0, total_cases=36. +- Insight: The CLI now has a full design ladder for the key workflows - parser, semantic summary, machine-usable plan, execution contract, and finally execution stub - which is a strong handoff point for wiring real handlers next. +- Next: raise the benchmark from stub shape to actual handler behavior or request payload construction for the same high-value workflows. + +## Segment 10: CLI Execution Previews +- New objective: raise the CLI benchmark from execution stubs to dry-run previews that expand the same high-value workflows into full previewable HTTP URLs or local-effect descriptions. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 25: baseline - cli_surface_score=85.714 (KEEP) +- Timestamp: 2026-03-26 19:27 +- What changed: Expanded the benchmark from execution stubs to dry-run previews, adding ignored tests for full preview URLs and human-readable local-effect descriptions across status, models scan/load, auth enable, throughput bench, and logs. +- Result: cli_surface_score=85.714, passed_cases=36, failed_cases=6, total_cases=42. +- Insight: The CLI already models execution stubs correctly, but `execution_preview()` still collapses every high-value workflow to `SummaryOnly { command: "preview-unimplemented" }`, so there is no user-facing dry-run layer yet. +- Next: map the highest-value execution stubs into concrete `HttpPreview` and `LocalPreview` values using the resolved `--server` base URL and readable local-effect text. + +### Run 26: add cli execution previews - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:28 +- What changed: Implemented concrete `execution_preview()` mapping in `crates/localllm-cli/src/main.rs`, expanding HTTP stubs into full URLs with preserved payloads and local actions into readable effect descriptions via `preview_effect()`. +- Result: cli_surface_score=100.0, passed_cases=42, failed_cases=0, total_cases=42. +- Insight: The CLI now has a complete dry-run preview layer for the highest-value workflows, which is a strong bridge from internal planning seams to future real handlers and `--dry-run` UX. +- Next: raise the benchmark from preview shape to actual handler behavior or request-body construction details for status, throughput, and model-management workflows. + +## Segment 11: CLI Dry-Run Output +- New objective: raise the CLI benchmark from internal preview objects to final user-facing `--dry-run` output in both `text` and `json`, so the same high-value workflows are ready for a real preview UX at the command line. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green. + +### Run 27: baseline - cli_surface_score=87.5 (KEEP) +- Timestamp: 2026-03-26 19:32 +- What changed: Expanded the benchmark from internal `ExecutionPreview` objects to final `--dry-run` UX in both text and JSON, while keeping `render_execution_preview()` intentionally shallow as `DRY RUN {summary}`. +- Result: cli_surface_score=87.5, passed_cases=42, failed_cases=6, total_cases=48. +- Insight: The CLI already knows how to build the preview object graph, but the final presentation layer still collapses to summaries instead of rendering health/chat HTTP previews and local-effect dry-run text. +- Next: render the existing preview layer into final text/json output using the current `ExecutionPreview` data rather than introducing a new execution seam. + +### Run 28: render cli dry-run previews - cli_surface_score=100.0 (KEEP) +- Timestamp: 2026-03-26 19:34 +- What changed: Implemented final `--dry-run` rendering in `crates/localllm-cli/src/main.rs`, mapping `ExecutionPreview` values to user-facing text and JSON output and reusing the existing preview pipeline (`plan_command` -> `execution_contract` -> `execution_stub` -> `execution_preview`). +- Result: cli_surface_score=100.0, passed_cases=48, failed_cases=0, total_cases=48. +- Insight: The CLI now has a complete dry-run stack for the highest-value workflows, from parser through final user-visible text/json previews, without yet executing real side effects. +- Next: raise the bar from dry-run rendering to real handler behavior or deterministic request/body construction for backend-facing commands. + +## Segment 12: CLI Backend Invocations +- New objective: raise the CLI benchmark from dry-run rendering to the first real backend-call specifications for commands that can already work today, especially `status`, `models list`, and `bench throughput`. +- Benchmark driver: `./autoresearch.sh` still runs ignored `localllm-cli` tests and reports `cli_surface_score`, `passed_cases`, `failed_cases`, and `total_cases`. +- Keep/discard rule remains test-driven: higher `cli_surface_score` wins while the normal non-ignored suites remain green.