merge branch 'claude/vibrant-austin' into main
# Conflicts: # autoresearch.jsonl # autoresearch.md # experiments/bench_tauri_runtime_surface.py # experiments/worklog.md # tauri-app/src-tauri/src/runtime/health.rs # tauri-app/src-tauri/src/runtime/mod.rs # tauri-app/src-tauri/src/runtime/types.rs # tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx
This commit is contained in:
commit
c91cbe51f2
10 files changed
+254
-6
No files matched your search
Submodule .claude/worktrees/vibrant-austin deleted from d78012373a.
@@ -31,3 +31,6 @@ nul
|
||||
.vscode/settings.json
|
||||
.idea/
|
||||
*.swp
|
||||
|
||||
# Local agent worktrees
|
||||
.claude/worktrees/
|
||||
@@ -62,3 +62,4 @@
|
||||
{"type":"config","name":"tauri-runtime-lifecycle-acceptance","metricName":"runtime_lifecycle_score","metricUnit":"%","bestDirection":"higher"}
|
||||
{"run":43,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1},"status":"keep","description":"align the Tauri lifecycle harness and keep the local CTA visible during mode switches","timestamp":1774585583,"segment":19}
|
||||
{"run":44,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1},"status":"keep","description":"cover saved local startup transitions and local onboarding from browse to ready","timestamp":1774586404,"segment":19}
|
||||
{"run":45,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1,"health_revalidation_passed":1},"status":"keep","description":"revalidate managed local health after startup and surface degraded state","timestamp":1774591917,"segment":19}
|
||||
+3
-2
@@ -10,7 +10,7 @@ CLI surface.
|
||||
|
||||
## Metrics
|
||||
- **Primary**: runtime_lifecycle_score (percent, higher is better)
|
||||
- **Secondary**: unit_passed, build_passed, cargo_check_passed, local_mode_cta_passed, saved_startup_flow_passed, local_onboarding_flow_passed
|
||||
- **Secondary**: unit_passed, build_passed, cargo_check_passed, local_mode_cta_passed, saved_startup_flow_passed, local_onboarding_flow_passed, health_revalidation_passed
|
||||
|
||||
## How to Run
|
||||
- `./autoresearch.sh`
|
||||
@@ -52,4 +52,5 @@ CLI surface.
|
||||
- The latest experiment adds app-surface smoke coverage for `RuntimeStatusCard` and `RuntimeSettingsDrawer`, then fixes a real regression in `chooseModel()` so a previously saved model path is preserved when a folder rescan returns no models. This keeps the unified drawer start action available for valid saved configs instead of silently disabling it.
|
||||
- The current experiment re-initializes the harness around the Tauri desktop surface by replacing the stale CLI benchmark with `experiments/bench_tauri_runtime_surface.py`, updating `autoresearch.sh` to score runtime-focused frontend tests plus build/check verification, and fixing a real drawer regression where switching from remote to local mode hid the primary local CTA until after save.
|
||||
- The latest experiment extends the runtime lifecycle acceptance contract with two hook-backed smoke checks: a saved local config must surface `stopped -> starting -> ready` through `useRuntimeManager`, and local onboarding via folder browse must auto-select a model, persist it, and reach a ready local runtime from the same drawer flow.
|
||||
- Current keep criteria for this segment are: the desktop runtime benchmark targets the active Tauri surface, switching into local mode keeps the apply/start CTA visible immediately, saved local startup exposes `starting` before `ready`, onboarding can browse/save/start from one local flow, and verification stays green across Vitest, TypeScript/Vite build, and Rust compilation.
|
||||
- The latest experiment adds periodic runtime truthfulness checks after startup: `refresh_process_state()` now revalidates `/health` while the managed child is still running, downgrades stale `ready` state to `degraded` when the process stops serving correctly, and restores `ready` after recovery. The desktop surface test already rendered degraded snapshots correctly; the missing correctness gap was backend state revalidation.
|
||||
- Current keep criteria for this segment are: the desktop runtime benchmark targets the active Tauri surface, switching into local mode keeps the apply/start CTA visible immediately, saved local startup exposes `starting` before `ready`, onboarding can browse/save/start from one local flow, stale managed processes demote from `ready` to `degraded` when `/health` stops reporting ready, and verification stays green across Vitest, TypeScript/Vite build, and Rust compilation.
|
||||
@@ -62,6 +62,14 @@ def main() -> int:
|
||||
"-t",
|
||||
"keeps the local primary CTA visible while switching from remote to local mode",
|
||||
]
|
||||
health_revalidation_surface_command = [
|
||||
command_name("npm"),
|
||||
"test",
|
||||
"--",
|
||||
"src/features/runtime/components/RuntimeSurface.test.tsx",
|
||||
"-t",
|
||||
"surfaces degraded local runtime state after periodic health revalidation fails",
|
||||
]
|
||||
build_command = [command_name("npm"), "run", "build"]
|
||||
cargo_check_command = [
|
||||
command_name("cargo"),
|
||||
@@ -69,27 +77,49 @@ def main() -> int:
|
||||
"--manifest-path",
|
||||
str(tauri_backend / "Cargo.toml"),
|
||||
]
|
||||
health_revalidation_backend_command = [
|
||||
command_name("cargo"),
|
||||
"test",
|
||||
"--manifest-path",
|
||||
str(tauri_backend / "Cargo.toml"),
|
||||
"runtime::tests::runtime_revalidation_marks_unhealthy_local_process_as_degraded",
|
||||
"--",
|
||||
"--exact",
|
||||
]
|
||||
|
||||
unit_code, _ = run_command(unit_command, tauri_app)
|
||||
saved_startup_code, _ = run_command(saved_startup_command, tauri_app)
|
||||
local_onboarding_code, _ = run_command(local_onboarding_command, tauri_app)
|
||||
local_mode_cta_code, _ = run_command(local_mode_cta_command, tauri_app)
|
||||
health_revalidation_surface_code, _ = run_command(
|
||||
health_revalidation_surface_command, tauri_app
|
||||
)
|
||||
build_code, _ = run_command(build_command, tauri_app)
|
||||
cargo_check_code, _ = run_command(cargo_check_command, root)
|
||||
health_revalidation_backend_code, _ = run_command(
|
||||
health_revalidation_backend_command, root
|
||||
)
|
||||
|
||||
unit_passed = 1 if unit_code == 0 else 0
|
||||
saved_startup_flow_passed = 1 if saved_startup_code == 0 else 0
|
||||
local_onboarding_flow_passed = 1 if local_onboarding_code == 0 else 0
|
||||
local_mode_cta_passed = 1 if local_mode_cta_code == 0 else 0
|
||||
health_revalidation_passed = (
|
||||
1
|
||||
if health_revalidation_surface_code == 0
|
||||
and health_revalidation_backend_code == 0
|
||||
else 0
|
||||
)
|
||||
build_passed = 1 if build_code == 0 else 0
|
||||
cargo_check_passed = 1 if cargo_check_code == 0 else 0
|
||||
|
||||
total_checks = 6
|
||||
total_checks = 7
|
||||
passed_checks = (
|
||||
unit_passed
|
||||
+ saved_startup_flow_passed
|
||||
+ local_onboarding_flow_passed
|
||||
+ local_mode_cta_passed
|
||||
+ health_revalidation_passed
|
||||
+ build_passed
|
||||
+ cargo_check_passed
|
||||
)
|
||||
@@ -100,6 +130,7 @@ def main() -> int:
|
||||
print(f"METRIC local_mode_cta_passed={local_mode_cta_passed}")
|
||||
print(f"METRIC saved_startup_flow_passed={saved_startup_flow_passed}")
|
||||
print(f"METRIC local_onboarding_flow_passed={local_onboarding_flow_passed}")
|
||||
print(f"METRIC health_revalidation_passed={health_revalidation_passed}")
|
||||
print(f"METRIC build_passed={build_passed}")
|
||||
print(f"METRIC cargo_check_passed={cargo_check_passed}")
|
||||
|
||||
|
||||
@@ -427,3 +427,10 @@
|
||||
- Result: runtime_lifecycle_score=100.0, unit_passed=1, build_passed=1, cargo_check_passed=1, local_mode_cta_passed=1, saved_startup_flow_passed=1, local_onboarding_flow_passed=1.
|
||||
- Insight: The saved local startup flow was previously trustworthy only after the backend returned; the desktop surface showed `Working...` but skipped the visible `starting` state entirely. Moving the in-flight state into `useRuntimeManager` closes that UX gap and makes the status card/drawer reflect the real lifecycle immediately. The onboarding smoke also proves the unified drawer now covers browse -> auto-select -> save -> start without falling back to separate legacy controls.
|
||||
- Next: keep the lifecycle segment focused on truthfulness after startup, especially periodic `/health` revalidation that can demote a stale `ready` process into `degraded` when the child stops serving correctly.
|
||||
|
||||
### Run 45: revalidate managed local health after startup and surface degraded state - runtime_lifecycle_score=100.0 (KEEP)
|
||||
- Timestamp: 2026-03-27 03:11
|
||||
- What changed: Added a degraded-state lifecycle check to `tauri-app/src/features/runtime/components/RuntimeSurface.test.tsx`, extended `experiments/bench_tauri_runtime_surface.py` with `health_revalidation_passed`, and updated `tauri-app/src-tauri/src/runtime/health.rs` plus `tauri-app/src-tauri/src/runtime/mod.rs` so a running managed local child is periodically revalidated via `/health`. The runtime now demotes stale `ready` state to `degraded` when the process is still alive but no longer serves a healthy endpoint, and clears the `health_check_failed` error when health recovers. Added Rust tests covering both the degrade and recovery paths.
|
||||
- Result: runtime_lifecycle_score=100.0, unit_passed=1, build_passed=1, cargo_check_passed=1, local_mode_cta_passed=1, saved_startup_flow_passed=1, local_onboarding_flow_passed=1, health_revalidation_passed=1.
|
||||
- Insight: The frontend surface was already honest once it received a degraded snapshot; the real lifecycle gap was that backend runtime state could stay stuck at `ready` forever as long as the child process had not exited. Revalidating `/health` inside runtime state refresh closes that truthfulness gap without reintroducing a second frontend control plane.
|
||||
- Next: this lifecycle segment is saturated again at `100.0%`. The next worthwhile move should raise the bar beyond state truthfulness, likely around a higher-fidelity desktop integration check or a real managed-runtime smoke path against an actual local model process.
|
||||
@@ -62,6 +62,31 @@ pub fn wait_for_health(addr: &str, timeout: Duration) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub fn check_health(addr: &str, timeout: Duration) -> bool {
|
||||
let Some(socket_addr) = resolve_socket_addr(addr) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
match TcpStream::connect_timeout(&socket_addr, timeout) {
|
||||
Ok(mut stream) => {
|
||||
let _ = stream.set_read_timeout(Some(timeout));
|
||||
let _ = stream.set_write_timeout(Some(timeout));
|
||||
|
||||
let request =
|
||||
format!("GET /health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n");
|
||||
|
||||
if stream.write_all(request.as_bytes()).is_ok() {
|
||||
let mut response = String::new();
|
||||
return stream.read_to_string(&mut response).is_ok()
|
||||
&& health_response_is_ready(&response);
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::wait_for_health;
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::time::Duration;
|
||||
use tauri::{AppHandle, State};
|
||||
|
||||
use self::config::{load_runtime_config, save_runtime_config};
|
||||
use self::health::wait_for_health;
|
||||
use self::health::{check_health, wait_for_health};
|
||||
use self::process::{find_localllm_binary, spawn_localllm};
|
||||
use self::types::{
|
||||
ConnectionMode, RuntimeConfig, RuntimeErrorInfo, RuntimeSnapshot, RuntimeStatus, StartPolicy,
|
||||
@@ -82,6 +82,28 @@ fn refresh_process_state(inner: &mut RuntimeInner) {
|
||||
if inner.status == RuntimeStatus::Stopped {
|
||||
inner.status = RuntimeStatus::Ready;
|
||||
}
|
||||
|
||||
if inner.config.connection_mode == ConnectionMode::Local
|
||||
&& matches!(inner.status, RuntimeStatus::Ready | RuntimeStatus::Degraded)
|
||||
{
|
||||
if check_health(&inner.config.addr, Duration::from_millis(400)) {
|
||||
inner.status = RuntimeStatus::Ready;
|
||||
if inner.last_error.as_ref().map(|error| error.code.as_str())
|
||||
== Some("health_check_failed")
|
||||
{
|
||||
inner.last_error = None;
|
||||
}
|
||||
} else {
|
||||
inner.status = RuntimeStatus::Degraded;
|
||||
inner.last_error = Some(runtime_error(
|
||||
"health_check_failed",
|
||||
format!(
|
||||
"Local API process is still running, but /health is no longer ready on {}.",
|
||||
inner.config.addr
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
inner.child = None;
|
||||
@@ -252,3 +274,116 @@ pub fn stop_runtime(state: State<'_, RuntimeState>) -> Result<RuntimeSnapshot, S
|
||||
inner.last_error = None;
|
||||
Ok(build_snapshot(&inner))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn spawn_healthy_server() -> String {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind healthy port");
|
||||
let addr = listener
|
||||
.local_addr()
|
||||
.expect("read healthy addr")
|
||||
.to_string();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
if let Ok((mut stream, _)) = listener.accept() {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let mut buffer = [0_u8; 1024];
|
||||
let _ = stream.read(&mut buffer);
|
||||
let _ = stream.write_all(
|
||||
b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 58\r\n\r\n{\"status\":\"ok\",\"model_loaded\":true,\"model_id\":\"demo\"}",
|
||||
);
|
||||
let _ = stream.flush();
|
||||
}
|
||||
});
|
||||
|
||||
addr
|
||||
}
|
||||
|
||||
fn unused_local_addr() -> String {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind unused port");
|
||||
let addr = listener.local_addr().expect("read local addr").to_string();
|
||||
drop(listener);
|
||||
addr
|
||||
}
|
||||
|
||||
fn spawn_sleeping_child() -> Child {
|
||||
std::process::Command::new("python")
|
||||
.args(["-c", "import time; time.sleep(60)"])
|
||||
.spawn()
|
||||
.expect("spawn sleeping child")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_revalidation_marks_unhealthy_local_process_as_degraded() {
|
||||
let mut child = spawn_sleeping_child();
|
||||
let addr = unused_local_addr();
|
||||
|
||||
let (status, error_code) = {
|
||||
let mut inner = RuntimeInner {
|
||||
child: Some(child),
|
||||
status: RuntimeStatus::Ready,
|
||||
last_error: None,
|
||||
config: RuntimeConfig {
|
||||
model_path: Some("D:/models/demo.gguf".to_string()),
|
||||
addr,
|
||||
..RuntimeConfig::default()
|
||||
},
|
||||
};
|
||||
|
||||
refresh_process_state(&mut inner);
|
||||
|
||||
child = inner.child.take().expect("process remains managed");
|
||||
|
||||
(
|
||||
inner.status,
|
||||
inner.last_error.as_ref().map(|error| error.code.clone()),
|
||||
)
|
||||
};
|
||||
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
|
||||
assert_eq!(status, RuntimeStatus::Degraded);
|
||||
assert_eq!(error_code.as_deref(), Some("health_check_failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_revalidation_restores_local_process_to_ready_after_health_recovers() {
|
||||
let mut child = spawn_sleeping_child();
|
||||
let addr = spawn_healthy_server();
|
||||
|
||||
let (status, error_code) = {
|
||||
let mut inner = RuntimeInner {
|
||||
child: Some(child),
|
||||
status: RuntimeStatus::Degraded,
|
||||
last_error: Some(runtime_error(
|
||||
"health_check_failed",
|
||||
"Local API process is still running, but /health is no longer ready.",
|
||||
)),
|
||||
config: RuntimeConfig {
|
||||
model_path: Some("D:/models/demo.gguf".to_string()),
|
||||
addr,
|
||||
..RuntimeConfig::default()
|
||||
},
|
||||
};
|
||||
|
||||
refresh_process_state(&mut inner);
|
||||
|
||||
child = inner.child.take().expect("process remains managed");
|
||||
|
||||
(
|
||||
inner.status,
|
||||
inner.last_error.as_ref().map(|error| error.code.clone()),
|
||||
)
|
||||
};
|
||||
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
|
||||
assert_eq!(status, RuntimeStatus::Ready);
|
||||
assert_eq!(error_code, None);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ pub enum StartPolicy {
|
||||
OnAppLaunch,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
|
||||
#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Default, Debug)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RuntimeStatus {
|
||||
#[default]
|
||||
|
||||
@@ -101,6 +101,7 @@ describe("Runtime app surface", () => {
|
||||
act(() => root.unmount());
|
||||
container.remove();
|
||||
vi.clearAllMocks();
|
||||
vi.useRealTimers();
|
||||
// @ts-expect-error React test environment flag
|
||||
globalThis.IS_REACT_ACT_ENVIRONMENT = false;
|
||||
});
|
||||
@@ -376,4 +377,49 @@ describe("Runtime app surface", () => {
|
||||
expect(container.textContent).toContain("Local API ready");
|
||||
expect(container.textContent).toContain("Stop local API");
|
||||
});
|
||||
|
||||
it("surfaces degraded local runtime state after periodic health revalidation fails", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const readySnapshot = makeSnapshot({
|
||||
status: "ready",
|
||||
ready: true,
|
||||
process_running: true,
|
||||
binary_path: "D:/bin/localllm.exe",
|
||||
config: {
|
||||
connection_mode: "local",
|
||||
model_path: "D:/models/demo.gguf",
|
||||
},
|
||||
});
|
||||
|
||||
const degradedSnapshot = makeSnapshot({
|
||||
...readySnapshot,
|
||||
status: "degraded",
|
||||
ready: false,
|
||||
last_error: {
|
||||
code: "health_check_failed",
|
||||
message: "Local API process is still running, but /health is no longer ready.",
|
||||
},
|
||||
});
|
||||
|
||||
vi.mocked(bootstrapRuntime).mockResolvedValue(readySnapshot);
|
||||
vi.mocked(getRuntimeSnapshot).mockResolvedValue(degradedSnapshot);
|
||||
|
||||
await act(async () => {
|
||||
root.render(<RuntimeManagerHarness />);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(container.textContent).toContain("Ready");
|
||||
expect(container.textContent).toContain("Stop");
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
});
|
||||
|
||||
expect(vi.mocked(getRuntimeSnapshot)).toHaveBeenCalledTimes(1);
|
||||
expect(container.textContent).toContain("Attention needed");
|
||||
expect(container.textContent).toContain("Local API process is still running, but /health is no longer ready.");
|
||||
expect(container.textContent).toContain("Stop");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user