46 lines
1.3 KiB
Bash
46 lines
1.3 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$SCRIPT_DIR"
|
|
PORT="${LOCALLLM_BENCH_PORT:-18080}"
|
|
HOST="127.0.0.1:${PORT}"
|
|
MODEL_PATH="${LOCALLLM_MODEL:-$ROOT/models/Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf}"
|
|
MODEL_ID="${LOCALLLM_BENCH_MODEL_ID:-$(basename "$MODEL_PATH")}"
|
|
MODEL_ID="${MODEL_ID%.gguf}"
|
|
SERVER_BIN="$ROOT/target/release/localllm.exe"
|
|
LOG_DIR="$ROOT/logs"
|
|
SERVER_LOG="$LOG_DIR/autoresearch-server.log"
|
|
|
|
mkdir -p "$LOG_DIR" "$ROOT/experiments"
|
|
|
|
if [[ ! -f "$MODEL_PATH" ]]; then
|
|
echo "[ERROR] Model file not found: $MODEL_PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() {
|
|
if [[ -n "${SERVER_PID:-}" ]]; then
|
|
kill "$SERVER_PID" >/dev/null 2>&1 || true
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
echo "[INFO] Fast preflight (cargo check)..." >&2
|
|
cargo check -p inference-server --quiet
|
|
|
|
echo "[INFO] Building release server..." >&2
|
|
cargo build --release -p inference-server --bin localllm --quiet
|
|
|
|
echo "[INFO] Starting server on $HOST ..." >&2
|
|
LOCALLLM_ADDR="$HOST" \
|
|
LOCALLLM_MODEL="$MODEL_PATH" \
|
|
"$SERVER_BIN" >"$SERVER_LOG" 2>&1 &
|
|
SERVER_PID=$!
|
|
|
|
python "$ROOT/experiments/bench_chat_latency.py" \
|
|
--base-url "http://$HOST" \
|
|
--model-id "$MODEL_ID" \
|
|
--server-log "$SERVER_LOG"
|