Initial ForgeLM import

experiments/bench_cli_surface.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", required=True)
+    args = parser.parse_args()
+
+    root = Path(args.root)
+    command = [
+        "cargo",
+        "test",
+        "-p",
+        "localllm-cli",
+        "cli_surface_",
+        "--",
+        "--ignored",
+        "--nocapture",
+    ]
+
+    completed = subprocess.run(
+        command,
+        cwd=root,
+        text=True,
+        capture_output=True,
+    )
+
+    output = (completed.stdout or "") + (completed.stderr or "")
+    sys.stdout.write(output)
+
+    passed_match = re.search(
+        r"test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
+    )
+    failed_match = re.search(
+        r"test result: FAILED\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
+    )
+
+    passed_cases = 0
+    failed_cases = 0
+    total_cases = 0
+
+    if passed_match:
+        passed_cases = int(passed_match.group(1))
+        failed_cases = int(passed_match.group(2))
+        total_cases = passed_cases + failed_cases
+    elif failed_match:
+        passed_cases = int(failed_match.group(1))
+        failed_cases = int(failed_match.group(2))
+        total_cases = passed_cases + failed_cases
+
+    if completed.returncode != 0 and total_cases == 0:
+        print("METRIC cli_surface_score=0.0")
+        print("METRIC passed_cases=0")
+        print("METRIC failed_cases=0")
+        print("METRIC total_cases=0")
+        return completed.returncode
+
+    score = 0.0 if total_cases == 0 else round((passed_cases / total_cases) * 100.0, 3)
+    print(f"METRIC cli_surface_score={score}")
+    print(f"METRIC passed_cases={passed_cases}")
+    print(f"METRIC failed_cases={failed_cases}")
+    print(f"METRIC total_cases={total_cases}")
+    return 0 if total_cases > 0 else completed.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())