#!/usr/bin/env python3 """ Hermaguard Benchmark — Score HermaGuard against known bug dataset. The benchmark does NOT run HermaGuard itself (it's a skill, not a binary). Instead, it scores JSON output that any harness produces. Usage: # 1. Run HermaGuard via your harness, capture JSON output per bug # (e.g., Claude Code: claude -p "/hermaguard --json" > results/bug-001.json) # (e.g., Codex CLI: codex exec --prompt "/hermaguard --json" > results/bug-001.json) # (e.g., Hermes Agent: run skill, save JSON output) # 2. Score the results against ground truth benchmark.py --score --results results/ --bugs bugs/ --output report.html # 3. Score a single bug benchmark.py --score --result results/bug-001.json --bug bugs/001-sql-injection The neutral path: any harness emits the --json report, then benchmark.py --score against ground truth. No assumption about which CLI or framework runs HermaGuard. """ import argparse import json import os import sys from datetime import datetime from pathlib import Path VERSION = "1.0.0" def discover_bugs(bugs_dir: str) -> list: """Find all bug directories containing ground-truth.json.""" bugs = [] for entry in sorted(Path(bugs_dir).iterdir()): if entry.is_dir() and (entry / "ground-truth.json").exists(): bugs.append(str(entry)) return bugs def load_bug(bug_dir: str) -> dict: """Load a bug's ground truth and metadata.""" gt_path = os.path.join(bug_dir, "ground-truth.json") readme_path = os.path.join(bug_dir, "README.md") with open(gt_path) as f: gt = json.load(f) readme = "" if os.path.exists(readme_path): with open(readme_path) as f: readme = f.read() return { "id": os.path.basename(bug_dir), "title": gt.get("title", os.path.basename(bug_dir)), "description": gt.get("description", ""), "severity": gt.get("severity", "UNKNOWN"), "category": gt.get("category", "unknown"), "expected_findings": gt.get("findings", []), "pre_diff": os.path.join(bug_dir, "pre.diff"), "post_diff": os.path.join(bug_dir, "post.diff"), "repo_path": os.path.join(bug_dir, "repo"), "readme": readme, } def _file_matches(exp_file: str, found_file: str) -> bool: """Deterministic file match on basename. Empty either side = pass-through.""" if not exp_file or not found_file: return True return os.path.basename(exp_file) == os.path.basename(found_file) def _class_matches(exp: dict, fnd: dict) -> bool: """Deterministic class match: both sides carry `classes` and they intersect.""" exp_classes = exp.get("classes") or [] fnd_classes = fnd.get("classes") or [] if not exp_classes or not fnd_classes: return False # not class-matchable — caller falls back to keywords return bool(set(exp_classes) & set(fnd_classes)) def score_run(bug: dict, run_info: dict) -> dict: """Score a run against ground truth. Matching is two-tier and deterministic: Tier 1 (preferred): class-based — ground truth entries carry `classes` (CWE/HG ids) and findings carry `classes`; a match requires a non-empty class intersection AND a basename file match. No thresholds, no fuzzy scores. Identical inputs always produce identical results. Tier 2 (legacy fallback): keyword overlap for old results that predate the classes field. Threshold fixed at 0.3, severity boost 1.3 — kept ONLY for backward compatibility; new ground truth must carry classes. """ expected = bug.get("expected_findings", []) found = run_info.get("findings", []) matched_expected = set() matched_found = set() matches = [] match_modes = {"class": 0, "keyword": 0} for i, exp in enumerate(expected): # ---- Tier 1: deterministic class match ---- if exp.get("classes"): for j, fnd in enumerate(found): if j in matched_found: continue if _class_matches(exp, fnd) and _file_matches(exp.get("file", ""), fnd.get("file", "")): matched_expected.add(i) matched_found.add(j) match_modes["class"] += 1 matches.append({ "expected_id": f"EXP-{i+1}", "found_id": fnd.get("id", f"HG-{j+1}"), "mode": "class", "matched_classes": sorted(set(exp["classes"]) & set(fnd.get("classes") or [])), "expected_severity": exp.get("severity", "?"), "found_severity": fnd.get("severity", "?"), "expected_file": exp.get("file", "?"), "found_file": fnd.get("file", "?"), }) break continue # class-carrying entries never fall through to keywords # ---- Tier 2: legacy keyword fallback ---- exp_text = f"{exp.get('file','')} {exp.get('trigger','')} {exp.get('description','')}".lower() best_score = 0 best_j = -1 for j, fnd in enumerate(found): if j in matched_found: continue fnd_text = f"{fnd.get('file','')} {fnd.get('trigger_condition','')} {fnd.get('consequence','')}".lower() # Simple keyword overlap score exp_words = set(exp_text.split()) fnd_words = set(fnd_text.split()) if not exp_words or not fnd_words: continue overlap = len(exp_words & fnd_words) / min(len(exp_words), len(fnd_words)) # Boost: same severity if exp.get("severity") == fnd.get("severity"): overlap *= 1.3 if overlap > best_score: best_score = overlap best_j = j if best_score > 0.3 and best_j >= 0: # threshold for a "match" matched_expected.add(i) matched_found.add(best_j) matched_fnd = found[best_j] match_modes["keyword"] += 1 matches.append({ "expected_id": f"EXP-{i+1}", "found_id": matched_fnd.get("id", f"HG-{best_j+1}"), "mode": "keyword", "score": round(best_score, 2), "expected_severity": exp.get("severity", "?"), "found_severity": matched_fnd.get("severity", "?"), "expected_file": exp.get("file", "?"), "found_file": matched_fnd.get("file", "?"), }) total_expected = len(expected) total_found = len(found) true_positives = len(matched_expected) false_negatives = total_expected - true_positives false_positives = total_found - len(matched_found) precision = round(true_positives / total_found * 100, 1) if total_found > 0 else 0 recall = round(true_positives / total_expected * 100, 1) if total_expected > 0 else 0 f1 = round(2 * precision * recall / (precision + recall), 1) if (precision + recall) > 0 else 0 # Severity accuracy severity_matches = 0 for m in matches: if m["expected_severity"] == m["found_severity"]: severity_matches += 1 severity_accuracy = round(severity_matches / len(matches) * 100, 1) if matches else 0 return { "bug_id": bug["id"], "bug_title": bug["title"], "bug_severity": bug["severity"], "bug_category": bug["category"], "expected_findings": total_expected, "found_findings": total_found, "true_positives": true_positives, "false_negatives": false_negatives, "false_positives": false_positives, "precision_pct": precision, "recall_pct": recall, "f1_score": f1, "severity_accuracy_pct": severity_accuracy, "matches": matches, "match_modes": match_modes, # A result file without an explicit run_ok flag is treated as a # completed run (backward compat) UNLESS it carries an error key — # a crashed run must never be scored as OK. "run_ok": bool(run_info.get("run_ok", not run_info.get("error"))), "run_error": run_info.get("error"), } def generate_report(scores: list, output_path: str): """Generate HTML benchmark report.""" total_bugs = len(scores) ok_runs = [s for s in scores if s["run_ok"]] avg_precision = round(sum(s["precision_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0 avg_recall = round(sum(s["recall_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0 avg_f1 = round(sum(s["f1_score"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0 avg_severity = round(sum(s["severity_accuracy_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0 total_fp = sum(s["false_positives"] for s in ok_runs) total_fn = sum(s["false_negatives"] for s in ok_runs) html = f""" HermaGuard Benchmark — {datetime.now().strftime('%d/%m/%Y')}

HermaGuard Benchmark

{datetime.now().strftime('%d/%m/%Y %H:%M')} · {total_bugs} bugs · v{VERSION}
{avg_f1}%
Avg F1 Score
{avg_recall}%
Avg Recall
{avg_precision}%
Avg Precision
{avg_severity}%
Severity Accuracy
{total_fp}
False Positives
{total_fn}
False Negatives

Results by Bug

""" for s in scores: if not s["run_ok"]: html += f"""\n""" continue pcls = "good" if s["precision_pct"] >= 70 else ("warn" if s["precision_pct"] >= 40 else "bad") rcls = "good" if s["recall_pct"] >= 70 else ("warn" if s["recall_pct"] >= 40 else "bad") fcls = "good" if s["f1_score"] >= 70 else ("warn" if s["f1_score"] >= 40 else "bad") html += f"""\n""" html += "
BugSeverityExpectedFoundTPFPFNPrecisionRecallF1Sev Acc
{s['bug_title'][:60]} ------ERROR: {s.get('run_error','Unknown')[:100]}
{s['bug_title'][:60]} {s['bug_severity']} {s['expected_findings']} {s['found_findings']} {s['true_positives']} {s['false_positives']} {s['false_negatives']} {s['precision_pct']}% {s['recall_pct']}% {s['f1_score']}% {s['severity_accuracy_pct']}%
\n" # Detailed matches for each bug html += "

Match Details

\n" for s in scores: if not s["run_ok"]: continue html += f"

{s['bug_title'][:80]} ({s['bug_severity']})

\n" if s["matches"]: html += "\n" for m in s["matches"]: sev_match = "✓" if m["expected_severity"] == m["found_severity"] else "✗" html += f"""\n""" html += "
ExpectedFoundSev MatchScore
{m['expected_file']} ({m['expected_severity']}) {m['found_file'][:50]} ({m['found_severity']}){sev_match}{m['score']}
\n" else: html += "

No matches — all findings missed or all expected bugs not found.

\n" if s.get("matches"): # Unmatched expected (FN) matched_exp_ids = {m["expected_id"] for m in s["matches"]} unmatched_exp = s.get("expected_findings_count", s["expected_findings"]) - len(matched_exp_ids) if unmatched_exp > 0: html += f"

{unmatched_exp} expected bug{'s' if unmatched_exp > 1 else ''} not found (false negative).

\n" html += f"""
HermaGuard Benchmark v{VERSION} · Generated {datetime.now().isoformat()}
""" with open(output_path, "w") as f: f.write(html) print(f"Report written to {output_path}") return output_path def main(): parser = argparse.ArgumentParser(description="HermaGuard Benchmark — score JSON output against ground truth") parser.add_argument("--bugs", default="bugs/", help="Path to bugs directory") parser.add_argument("--bug", help="Score a single bug by path") parser.add_argument("--results", default="results/", help="Directory containing per-bug JSON result files") parser.add_argument("--result", help="Single result JSON file (use with --bug)") parser.add_argument("--output", default="report.html", help="HTML report output path") parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}") args = parser.parse_args() if args.bug and args.result: # Single bug mode bug = load_bug(args.bug) with open(args.result) as f: run_info = json.load(f) s = score_run(bug, run_info) status = "✓" if s["run_ok"] else "✗" print(f" {status} {bug['title'][:60]} — P:{s['precision_pct']}% R:{s['recall_pct']}% F1:{s['f1_score']}%") return bugs = discover_bugs(args.bugs) if not bugs: print(f"No bugs found in {args.bugs}") sys.exit(1) print(f"Found {len(bugs)} bug(s)") scores = [] for bug_dir in bugs: bug = load_bug(bug_dir) result_path = os.path.join(args.results, f"{bug['id']}.json") if os.path.exists(result_path): with open(result_path) as f: run_info = json.load(f) s = score_run(bug, run_info) scores.append(s) status = "✓" if s["run_ok"] else "✗" print(f" {status} {bug['title'][:60]} — P:{s['precision_pct']}% R:{s['recall_pct']}% F1:{s['f1_score']}%") else: scores.append({ "bug_id": bug["id"], "bug_title": bug["title"], "bug_severity": bug["severity"], "bug_category": bug["category"], "run_ok": False, "run_error": f"No results file at {result_path}", "precision_pct": 0, "recall_pct": 0, "f1_score": 0, "expected_findings": len(bug["expected_findings"]), "found_findings": 0, "true_positives": 0, "false_positives": 0, "false_negatives": len(bug["expected_findings"]), "severity_accuracy_pct": 0, "matches": [], }) # Generate report. --output is taken literally (path doubling bug fixed); # create the parent dir so an explicit path like results/report.html works. report_path = args.output parent = os.path.dirname(report_path) if parent: os.makedirs(parent, exist_ok=True) generate_report(scores, report_path) ok = [s for s in scores if s["run_ok"]] if ok: avg_f1 = round(sum(s["f1_score"] for s in ok) / len(ok), 1) print(f"\nOverall: {len(ok)}/{len(scores)} runs OK, avg F1: {avg_f1}%") else: print(f"\nAll runs lack result files. Run HermaGuard via your harness first, then score.") print(f"Example: run '/hermaguard --json' under any harness, save output to results/.json") print(f"Report: {report_path}") if __name__ == "__main__": main()