71 lines
1.6 KiB
Python
71 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
TEST_FILTER = "dynamic_template_"
|
|
TEST_PATTERN = re.compile(
|
|
r"^test\s+(.+dynamic_template_[^\s]+)\s+\.\.\.\s+(ok|FAILED)$"
|
|
)
|
|
|
|
|
|
def run_suite(root: Path) -> tuple[int, int, int]:
|
|
command = [
|
|
"cargo",
|
|
"test",
|
|
"-p",
|
|
"inference-server",
|
|
TEST_FILTER,
|
|
"--",
|
|
"--ignored",
|
|
"--nocapture",
|
|
]
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=root,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
|
|
output = completed.stdout + completed.stderr
|
|
print(output, end="")
|
|
|
|
seen: dict[str, str] = {}
|
|
for line in output.splitlines():
|
|
match = TEST_PATTERN.match(line.strip())
|
|
if match:
|
|
seen[match.group(1)] = match.group(2)
|
|
|
|
if not seen and completed.returncode != 0:
|
|
raise SystemExit(completed.returncode)
|
|
|
|
total = len(seen)
|
|
passed = sum(1 for status in seen.values() if status == "ok")
|
|
failed = total - passed
|
|
score = round((passed / total) * 100.0, 3) if total else 0.0
|
|
|
|
print(f"METRIC template_score={score}")
|
|
print(f"METRIC passed_cases={passed}")
|
|
print(f"METRIC failed_cases={failed}")
|
|
print(f"METRIC total_cases={total}")
|
|
return passed, failed, total
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--root", required=True)
|
|
args = parser.parse_args()
|
|
|
|
root = Path(args.root)
|
|
passed, failed, total = run_suite(root)
|
|
return 0 if total > 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|