73 lines
1.9 KiB
Python
73 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--root", required=True)
|
|
args = parser.parse_args()
|
|
|
|
root = Path(args.root)
|
|
command = [
|
|
"cargo",
|
|
"test",
|
|
"-p",
|
|
"localllm-cli",
|
|
"cli_surface_",
|
|
"--",
|
|
"--ignored",
|
|
"--nocapture",
|
|
]
|
|
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=root,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
output = (completed.stdout or "") + (completed.stderr or "")
|
|
sys.stdout.write(output)
|
|
|
|
passed_match = re.search(
|
|
r"test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
|
|
)
|
|
failed_match = re.search(
|
|
r"test result: FAILED\. (\d+) passed; (\d+) failed; (\d+) ignored;", output
|
|
)
|
|
|
|
passed_cases = 0
|
|
failed_cases = 0
|
|
total_cases = 0
|
|
|
|
if passed_match:
|
|
passed_cases = int(passed_match.group(1))
|
|
failed_cases = int(passed_match.group(2))
|
|
total_cases = passed_cases + failed_cases
|
|
elif failed_match:
|
|
passed_cases = int(failed_match.group(1))
|
|
failed_cases = int(failed_match.group(2))
|
|
total_cases = passed_cases + failed_cases
|
|
|
|
if completed.returncode != 0 and total_cases == 0:
|
|
print("METRIC cli_surface_score=0.0")
|
|
print("METRIC passed_cases=0")
|
|
print("METRIC failed_cases=0")
|
|
print("METRIC total_cases=0")
|
|
return completed.returncode
|
|
|
|
score = 0.0 if total_cases == 0 else round((passed_cases / total_cases) * 100.0, 3)
|
|
print(f"METRIC cli_surface_score={score}")
|
|
print(f"METRIC passed_cases={passed_cases}")
|
|
print(f"METRIC failed_cases={failed_cases}")
|
|
print(f"METRIC total_cases={total_cases}")
|
|
return 0 if total_cases > 0 else completed.returncode
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|