chore: fix autoresearch benchmark model id

This commit is contained in:
Evandro Fonseca Junior committed 2026-03-26 17:31:35 -03:00
1 parent f83e8b83f6
commit b38c77f875
2 files changed
+24 -19

No files matched your search

+21 -19
View File
@@ -9,21 +9,6 @@ import urllib.error
import urllib.request
PAYLOAD = {
"model": "local-model",
"stream": False,
"temperature": 0.0,
"top_p": 1.0,
"max_completion_tokens": 48,
"messages": [
{
"role": "user",
"content": "Write a compact Rust function that parses a comma-separated list of u32 values and returns a Vec<u32> while ignoring empty segments.",
}
],
}
def percentile(sorted_values, pct):
if not sorted_values:
return 0.0
@@ -63,8 +48,24 @@ def wait_for_health(base_url, timeout_s, server_log):
sys.exit(1)
def run_once(base_url):
body = json.dumps(PAYLOAD).encode("utf-8")
def build_payload(model_id):
return {
"model": model_id,
"stream": False,
"temperature": 0.0,
"top_p": 1.0,
"max_completion_tokens": 48,
"messages": [
{
"role": "user",
"content": "Write a compact Rust function that parses a comma-separated list of u32 values and returns a Vec<u32> while ignoring empty segments.",
}
],
}
def run_once(base_url, model_id):
body = json.dumps(build_payload(model_id)).encode("utf-8")
req = urllib.request.Request(
base_url + "/v1/chat/completions",
data=body,
@@ -82,6 +83,7 @@ def run_once(base_url):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--model-id", required=True)
parser.add_argument("--server-log", default="")
parser.add_argument("--warmups", type=int, default=1)
parser.add_argument("--samples", type=int, default=7)
@@ -91,12 +93,12 @@ def main():
wait_for_health(args.base_url, args.health_timeout, args.server_log)
for _ in range(args.warmups):
run_once(args.base_url)
run_once(args.base_url, args.model_id)
latencies = []
tps_values = []
for _ in range(args.samples):
latency_ms, tps = run_once(args.base_url)
latency_ms, tps = run_once(args.base_url, args.model_id)
latencies.append(latency_ms)
tps_values.append(tps)