chore: fix autoresearch benchmark model id
This commit is contained in:
1 parent
f83e8b83f6
commit
b38c77f875
2 files changed
+24
-19
No files matched your search
@@ -9,21 +9,6 @@ import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
PAYLOAD = {
|
||||
"model": "local-model",
|
||||
"stream": False,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"max_completion_tokens": 48,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Write a compact Rust function that parses a comma-separated list of u32 values and returns a Vec<u32> while ignoring empty segments.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def percentile(sorted_values, pct):
|
||||
if not sorted_values:
|
||||
return 0.0
|
||||
@@ -63,8 +48,24 @@ def wait_for_health(base_url, timeout_s, server_log):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run_once(base_url):
|
||||
body = json.dumps(PAYLOAD).encode("utf-8")
|
||||
def build_payload(model_id):
|
||||
return {
|
||||
"model": model_id,
|
||||
"stream": False,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"max_completion_tokens": 48,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Write a compact Rust function that parses a comma-separated list of u32 values and returns a Vec<u32> while ignoring empty segments.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_once(base_url, model_id):
|
||||
body = json.dumps(build_payload(model_id)).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
base_url + "/v1/chat/completions",
|
||||
data=body,
|
||||
@@ -82,6 +83,7 @@ def run_once(base_url):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--model-id", required=True)
|
||||
parser.add_argument("--server-log", default="")
|
||||
parser.add_argument("--warmups", type=int, default=1)
|
||||
parser.add_argument("--samples", type=int, default=7)
|
||||
@@ -91,12 +93,12 @@ def main():
|
||||
wait_for_health(args.base_url, args.health_timeout, args.server_log)
|
||||
|
||||
for _ in range(args.warmups):
|
||||
run_once(args.base_url)
|
||||
run_once(args.base_url, args.model_id)
|
||||
|
||||
latencies = []
|
||||
tps_values = []
|
||||
for _ in range(args.samples):
|
||||
latency_ms, tps = run_once(args.base_url)
|
||||
latency_ms, tps = run_once(args.base_url, args.model_id)
|
||||
latencies.append(latency_ms)
|
||||
tps_values.append(tps)
|
||||
|
||||
|
||||
Reference in new issue
Block a user