diff --git a/.claude/worktrees/vibrant-austin b/.claude/worktrees/vibrant-austin deleted file mode 160000 index d780123..0000000 --- a/.claude/worktrees/vibrant-austin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d78012373a29fddce98b4352aeedba87a990e60b diff --git a/.gitignore b/.gitignore index 1b16ec5..ac62c60 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ nul .vscode/settings.json .idea/ *.swp + +# Local agent worktrees +.claude/worktrees/ diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 50e757b..2055ff9 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -62,3 +62,4 @@ {"type":"config","name":"tauri-runtime-lifecycle-acceptance","metricName":"runtime_lifecycle_score","metricUnit":"%","bestDirection":"higher"} {"run":43,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1},"status":"keep","description":"align the Tauri lifecycle harness and keep the local CTA visible during mode switches","timestamp":1774585583,"segment":19} {"run":44,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1},"status":"keep","description":"cover saved local startup transitions and local onboarding from browse to ready","timestamp":1774586404,"segment":19} +{"run":45,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1,"health_revalidation_passed":1},"status":"keep","description":"revalidate managed local health after startup and surface degraded state","timestamp":1774591917,"segment":19} diff --git a/autoresearch.md b/autoresearch.md index 5ed8db1..b60e294 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -10,7 +10,7 @@ CLI surface. ## Metrics - **Primary**: runtime_lifecycle_score (percent, higher is better) -- **Secondary**: unit_passed, build_passed, cargo_check_passed, local_mode_cta_passed, saved_startup_flow_passed, local_onboarding_flow_passed +- **Secondary**: unit_passed, build_passed, cargo_check_passed, local_mode_cta_passed, saved_startup_flow_passed, local_onboarding_flow_passed, health_revalidation_passed ## How to Run - `./autoresearch.sh` @@ -52,4 +52,5 @@ CLI surface. - The latest experiment adds app-surface smoke coverage for `RuntimeStatusCard` and `RuntimeSettingsDrawer`, then fixes a real regression in `chooseModel()` so a previously saved model path is preserved when a folder rescan returns no models. This keeps the unified drawer start action available for valid saved configs instead of silently disabling it. - The current experiment re-initializes the harness around the Tauri desktop surface by replacing the stale CLI benchmark with `experiments/bench_tauri_runtime_surface.py`, updating `autoresearch.sh` to score runtime-focused frontend tests plus build/check verification, and fixing a real drawer regression where switching from remote to local mode hid the primary local CTA until after save. - The latest experiment extends the runtime lifecycle acceptance contract with two hook-backed smoke checks: a saved local config must surface `stopped -> starting -> ready` through `useRuntimeManager`, and local onboarding via folder browse must auto-select a model, persist it, and reach a ready local runtime from the same drawer flow. -- Current keep criteria for this segment are: the desktop runtime benchmark targets the active Tauri surface, switching into local mode keeps the apply/start CTA visible immediately, saved local startup exposes `starting` before `ready`, onboarding can browse/save/start from one local flow, and verification stays green across Vitest, TypeScript/Vite build, and Rust compilation. +- The latest experiment adds periodic runtime truthfulness checks after startup: `refresh_process_state()` now revalidates `/health` while the managed child is still running, downgrades stale `ready` state to `degraded` when the process stops serving correctly, and restores `ready` after recovery. The desktop surface test already rendered degraded snapshots correctly; the missing correctness gap was backend state revalidation. +- Current keep criteria for this segment are: the desktop runtime benchmark targets the active Tauri surface, switching into local mode keeps the apply/start CTA visible immediately, saved local startup exposes `starting` before `ready`, onboarding can browse/save/start from one local flow, stale managed processes demote from `ready` to `degraded` when `/health` stops reporting ready, and verification stays green across Vitest, TypeScript/Vite build, and Rust compilation. diff --git a/experiments/bench_tauri_runtime_surface.py b/experiments/bench_tauri_runtime_surface.py index 5fb85e6..1a192e2 100644 --- a/experiments/bench_tauri_runtime_surface.py +++ b/experiments/bench_tauri_runtime_surface.py @@ -62,6 +62,14 @@ def main() -> int: "-t", "keeps the local primary CTA visible while switching from remote to local mode", ] + health_revalidation_surface_command = [ + command_name("npm"), + "test", + "--", + "src/features/runtime/components/RuntimeSurface.test.tsx", + "-t", + "surfaces degraded local runtime state after periodic health revalidation fails", + ] build_command = [command_name("npm"), "run", "build"] cargo_check_command = [ command_name("cargo"), @@ -69,27 +77,49 @@ def main() -> int: "--manifest-path", str(tauri_backend / "Cargo.toml"), ] + health_revalidation_backend_command = [ + command_name("cargo"), + "test", + "--manifest-path", + str(tauri_backend / "Cargo.toml"), + "runtime::tests::runtime_revalidation_marks_unhealthy_local_process_as_degraded", + "--", + "--exact", + ] unit_code, _ = run_command(unit_command, tauri_app) saved_startup_code, _ = run_command(saved_startup_command, tauri_app) local_onboarding_code, _ = run_command(local_onboarding_command, tauri_app) local_mode_cta_code, _ = run_command(local_mode_cta_command, tauri_app) + health_revalidation_surface_code, _ = run_command( + health_revalidation_surface_command, tauri_app + ) build_code, _ = run_command(build_command, tauri_app) cargo_check_code, _ = run_command(cargo_check_command, root) + health_revalidation_backend_code, _ = run_command( + health_revalidation_backend_command, root + ) unit_passed = 1 if unit_code == 0 else 0 saved_startup_flow_passed = 1 if saved_startup_code == 0 else 0 local_onboarding_flow_passed = 1 if local_onboarding_code == 0 else 0 local_mode_cta_passed = 1 if local_mode_cta_code == 0 else 0 + health_revalidation_passed = ( + 1 + if health_revalidation_surface_code == 0 + and health_revalidation_backend_code == 0 + else 0 + ) build_passed = 1 if build_code == 0 else 0 cargo_check_passed = 1 if cargo_check_code == 0 else 0 - total_checks = 6 + total_checks = 7 passed_checks = ( unit_passed + saved_startup_flow_passed + local_onboarding_flow_passed + local_mode_cta_passed + + health_revalidation_passed + build_passed + cargo_check_passed ) @@ -100,6 +130,7 @@ def main() -> int: print(f"METRIC local_mode_cta_passed={local_mode_cta_passed}") print(f"METRIC saved_startup_flow_passed={saved_startup_flow_passed}") print(f"METRIC local_onboarding_flow_passed={local_onboarding_flow_passed}") + print(f"METRIC health_revalidation_passed={health_revalidation_passed}") print(f"METRIC build_passed={build_passed}") print(f"METRIC cargo_check_passed={cargo_check_passed}") diff --git a/experiments/worklog.md b/experiments/worklog.md index c9face4..85f949b 100644 --- a/experiments/worklog.md +++ b/experiments/worklog.md @@ -427,3 +427,10 @@ - Result: runtime_lifecycle_score=100.0, unit_passed=1, build_passed=1, cargo_check_passed=1, local_mode_cta_passed=1, saved_startup_flow_passed=1, local_onboarding_flow_passed=1. - Insight: The saved local startup flow was previously trustworthy only after the backend returned; the desktop surface showed `Working...` but skipped the visible `starting` state entirely. Moving the in-flight state into `useRuntimeManager` closes that UX gap and makes the status card/drawer reflect the real lifecycle immediately. The onboarding smoke also proves the unified drawer now covers browse -> auto-select -> save -> start without falling back to separate legacy controls. - Next: keep the lifecycle segment focused on truthfulness after startup, especially periodic `/health` revalidation that can demote a stale `ready` process into `degraded` when the child stops serving correctly. + +### Run 45: revalidate managed local health after startup and surface degraded state - runtime_lifecycle_score=100.0 (KEEP) +- Timestamp: 2026-03-27 03:11 +- What changed: Added a degraded-state lifecycle check to `tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx`, extended `experiments/bench_tauri_runtime_surface.py` with `health_revalidation_passed`, and updated `tauri-app/src-tauri/src/runtime/health.rs` plus `tauri-app/src-tauri/src/runtime/mod.rs` so a running managed local child is periodically revalidated via `/health`. The runtime now demotes stale `ready` state to `degraded` when the process is still alive but no longer serves a healthy endpoint, and clears the `health_check_failed` error when health recovers. Added Rust tests covering both the degrade and recovery paths. +- Result: runtime_lifecycle_score=100.0, unit_passed=1, build_passed=1, cargo_check_passed=1, local_mode_cta_passed=1, saved_startup_flow_passed=1, local_onboarding_flow_passed=1, health_revalidation_passed=1. +- Insight: The frontend surface was already honest once it received a degraded snapshot; the real lifecycle gap was that backend runtime state could stay stuck at `ready` forever as long as the child process had not exited. Revalidating `/health` inside runtime state refresh closes that truthfulness gap without reintroducing a second frontend control plane. +- Next: this lifecycle segment is saturated again at `100.0%`. The next worthwhile move should raise the bar beyond state truthfulness, likely around a higher-fidelity desktop integration check or a real managed-runtime smoke path against an actual local model process. diff --git a/tauri-app/src-tauri/src/runtime/health.rs b/tauri-app/src-tauri/src/runtime/health.rs index f6464c1..c25d546 100644 --- a/tauri-app/src-tauri/src/runtime/health.rs +++ b/tauri-app/src-tauri/src/runtime/health.rs @@ -62,6 +62,31 @@ pub fn wait_for_health(addr: &str, timeout: Duration) -> bool { false } +pub fn check_health(addr: &str, timeout: Duration) -> bool { + let Some(socket_addr) = resolve_socket_addr(addr) else { + return false; + }; + + match TcpStream::connect_timeout(&socket_addr, timeout) { + Ok(mut stream) => { + let _ = stream.set_read_timeout(Some(timeout)); + let _ = stream.set_write_timeout(Some(timeout)); + + let request = + format!("GET /health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n"); + + if stream.write_all(request.as_bytes()).is_ok() { + let mut response = String::new(); + return stream.read_to_string(&mut response).is_ok() + && health_response_is_ready(&response); + } + + false + } + Err(_) => false, + } +} + #[cfg(test)] mod tests { use super::wait_for_health; diff --git a/tauri-app/src-tauri/src/runtime/mod.rs b/tauri-app/src-tauri/src/runtime/mod.rs index 5b2ec67..bcd06c5 100644 --- a/tauri-app/src-tauri/src/runtime/mod.rs +++ b/tauri-app/src-tauri/src/runtime/mod.rs @@ -10,7 +10,7 @@ use std::time::Duration; use tauri::{AppHandle, State}; use self::config::{load_runtime_config, save_runtime_config}; -use self::health::wait_for_health; +use self::health::{check_health, wait_for_health}; use self::process::{find_localllm_binary, spawn_localllm}; use self::types::{ ConnectionMode, RuntimeConfig, RuntimeErrorInfo, RuntimeSnapshot, RuntimeStatus, StartPolicy, @@ -82,6 +82,28 @@ fn refresh_process_state(inner: &mut RuntimeInner) { if inner.status == RuntimeStatus::Stopped { inner.status = RuntimeStatus::Ready; } + + if inner.config.connection_mode == ConnectionMode::Local + && matches!(inner.status, RuntimeStatus::Ready | RuntimeStatus::Degraded) + { + if check_health(&inner.config.addr, Duration::from_millis(400)) { + inner.status = RuntimeStatus::Ready; + if inner.last_error.as_ref().map(|error| error.code.as_str()) + == Some("health_check_failed") + { + inner.last_error = None; + } + } else { + inner.status = RuntimeStatus::Degraded; + inner.last_error = Some(runtime_error( + "health_check_failed", + format!( + "Local API process is still running, but /health is no longer ready on {}.", + inner.config.addr + ), + )); + } + } } Err(error) => { inner.child = None; @@ -252,3 +274,116 @@ pub fn stop_runtime(state: State<'_, RuntimeState>) -> Result String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind healthy port"); + let addr = listener + .local_addr() + .expect("read healthy addr") + .to_string(); + + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + use std::io::{Read, Write}; + + let mut buffer = [0_u8; 1024]; + let _ = stream.read(&mut buffer); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 58\r\n\r\n{\"status\":\"ok\",\"model_loaded\":true,\"model_id\":\"demo\"}", + ); + let _ = stream.flush(); + } + }); + + addr + } + + fn unused_local_addr() -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind unused port"); + let addr = listener.local_addr().expect("read local addr").to_string(); + drop(listener); + addr + } + + fn spawn_sleeping_child() -> Child { + std::process::Command::new("python") + .args(["-c", "import time; time.sleep(60)"]) + .spawn() + .expect("spawn sleeping child") + } + + #[test] + fn runtime_revalidation_marks_unhealthy_local_process_as_degraded() { + let mut child = spawn_sleeping_child(); + let addr = unused_local_addr(); + + let (status, error_code) = { + let mut inner = RuntimeInner { + child: Some(child), + status: RuntimeStatus::Ready, + last_error: None, + config: RuntimeConfig { + model_path: Some("D:/models/demo.gguf".to_string()), + addr, + ..RuntimeConfig::default() + }, + }; + + refresh_process_state(&mut inner); + + child = inner.child.take().expect("process remains managed"); + + ( + inner.status, + inner.last_error.as_ref().map(|error| error.code.clone()), + ) + }; + + let _ = child.kill(); + let _ = child.wait(); + + assert_eq!(status, RuntimeStatus::Degraded); + assert_eq!(error_code.as_deref(), Some("health_check_failed")); + } + + #[test] + fn runtime_revalidation_restores_local_process_to_ready_after_health_recovers() { + let mut child = spawn_sleeping_child(); + let addr = spawn_healthy_server(); + + let (status, error_code) = { + let mut inner = RuntimeInner { + child: Some(child), + status: RuntimeStatus::Degraded, + last_error: Some(runtime_error( + "health_check_failed", + "Local API process is still running, but /health is no longer ready.", + )), + config: RuntimeConfig { + model_path: Some("D:/models/demo.gguf".to_string()), + addr, + ..RuntimeConfig::default() + }, + }; + + refresh_process_state(&mut inner); + + child = inner.child.take().expect("process remains managed"); + + ( + inner.status, + inner.last_error.as_ref().map(|error| error.code.clone()), + ) + }; + + let _ = child.kill(); + let _ = child.wait(); + + assert_eq!(status, RuntimeStatus::Ready); + assert_eq!(error_code, None); + } +} diff --git a/tauri-app/src-tauri/src/runtime/types.rs b/tauri-app/src-tauri/src/runtime/types.rs index 52e7122..e486ff9 100644 --- a/tauri-app/src-tauri/src/runtime/types.rs +++ b/tauri-app/src-tauri/src/runtime/types.rs @@ -16,7 +16,7 @@ pub enum StartPolicy { OnAppLaunch, } -#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Default, Debug)] #[serde(rename_all = "snake_case")] pub enum RuntimeStatus { #[default] diff --git a/tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx b/tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx index 1be78f7..af44e6e 100644 --- a/tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx +++ b/tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx @@ -101,6 +101,7 @@ describe("Runtime app surface", () => { act(() => root.unmount()); container.remove(); vi.clearAllMocks(); + vi.useRealTimers(); // @ts-expect-error React test environment flag globalThis.IS_REACT_ACT_ENVIRONMENT = false; }); @@ -376,4 +377,49 @@ describe("Runtime app surface", () => { expect(container.textContent).toContain("Local API ready"); expect(container.textContent).toContain("Stop local API"); }); + + it("surfaces degraded local runtime state after periodic health revalidation fails", async () => { + vi.useFakeTimers(); + + const readySnapshot = makeSnapshot({ + status: "ready", + ready: true, + process_running: true, + binary_path: "D:/bin/localllm.exe", + config: { + connection_mode: "local", + model_path: "D:/models/demo.gguf", + }, + }); + + const degradedSnapshot = makeSnapshot({ + ...readySnapshot, + status: "degraded", + ready: false, + last_error: { + code: "health_check_failed", + message: "Local API process is still running, but /health is no longer ready.", + }, + }); + + vi.mocked(bootstrapRuntime).mockResolvedValue(readySnapshot); + vi.mocked(getRuntimeSnapshot).mockResolvedValue(degradedSnapshot); + + await act(async () => { + root.render(); + await Promise.resolve(); + }); + + expect(container.textContent).toContain("Ready"); + expect(container.textContent).toContain("Stop"); + + await act(async () => { + await vi.advanceTimersByTimeAsync(3000); + }); + + expect(vi.mocked(getRuntimeSnapshot)).toHaveBeenCalledTimes(1); + expect(container.textContent).toContain("Attention needed"); + expect(container.textContent).toContain("Local API process is still running, but /health is no longer ready."); + expect(container.textContent).toContain("Stop"); + }); });