feat: vendorizar as 10 skills do pilar de seguranca em skills/

Copia local (nao fork, nao submodule) de safedeps, sca-audit, secrets-scan, code-review-security, api-security-review, web-security-review, cve-triage, patch-prioritization, dependency-scanning e hermaguard -- commit exato de origem e licenca de cada uma em skills/NOTICE.md. Elimina o passo de clone externo pro pilar de seguranca do wf-security-review: git clone deste repo ja traz tudo funcional. Corrige tambem o comando de instalacao pro Claude Code nos dois READMEs e no docs/SKILLS.md: Claude Code exige um diretorio por skill (skills/<nome>/SKILL.md), nao aceita os .md soltos que o comando antigo copiava -- confirmado testando a instalacao real.
2026-08-24 00:32:20 -03:00
1 parent ee44fa0f7a
commit c17192c9fe
@@ -0,0 +1,33 @@
+# Hermaguard Benchmark
+
+Score-only. It does **not** run HermaGuard (that's a skill, not a binary). Your
+harness produces a `--json` report per bug; this scores it against ground truth.
+
+## Layout
+
+- `bugs/<id>/` — one case per planted bug: `code/app.py` (fixture, **no answer-key
+  comments**), `pre.diff` (the change that introduced it), `ground-truth.json`
+  (the findings a correct review should surface).
+- `results/<id>.json` — a harness's review output for that bug (the shape the
+  scorer reads). The committed ones are a **reference run**, see below.
+
+## Run
+
+```bash
+python3 benchmark.py --bugs bugs --results results --output results/report.html
+```
+
+## Reference run (committed in `results/`)
+
+A single `claude-opus-4-8` harness reviewed the 5 fixtures. **Not blind:** the
+corpus was authored and reviewed by the same agent, so recall here is an upper
+bound. The corpus exists for blind runs by any harness, seed bugs authored by
+someone other than the reviewer for an independent number.
+
+| | Recall | Precision | F1 |
+|---|---|---|---|
+| 5 bugs | 100% | 90% | 93.3% |
+
+The one sub-100% precision (bug 001) is a real out-of-ground-truth finding (an
+unclosed DB connection), not a hallucination. Each result file carries a
+`provenance` block (model, mode, blind flag).
@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+"""
+Hermaguard Benchmark — Score HermaGuard against known bug dataset.
+
+The benchmark does NOT run HermaGuard itself (it's a skill, not a binary).
+Instead, it scores JSON output that any harness produces.
+
+Usage:
+    # 1. Run HermaGuard via your harness, capture JSON output per bug
+    #    (e.g., Claude Code: claude -p "/hermaguard --json" > results/bug-001.json)
+    #    (e.g., Codex CLI: codex exec --prompt "/hermaguard --json" > results/bug-001.json)
+    #    (e.g., Hermes Agent: run skill, save JSON output)
+
+    # 2. Score the results against ground truth
+    benchmark.py --score --results results/ --bugs bugs/ --output report.html
+
+    # 3. Score a single bug
+    benchmark.py --score --result results/bug-001.json --bug bugs/001-sql-injection
+
+The neutral path: any harness emits the --json report, then benchmark.py --score
+against ground truth. No assumption about which CLI or framework runs HermaGuard.
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+VERSION = "1.0.0"
+
+
+def discover_bugs(bugs_dir: str) -> list:
+    """Find all bug directories containing ground-truth.json."""
+    bugs = []
+    for entry in sorted(Path(bugs_dir).iterdir()):
+        if entry.is_dir() and (entry / "ground-truth.json").exists():
+            bugs.append(str(entry))
+    return bugs
+
+
+def load_bug(bug_dir: str) -> dict:
+    """Load a bug's ground truth and metadata."""
+    gt_path = os.path.join(bug_dir, "ground-truth.json")
+    readme_path = os.path.join(bug_dir, "README.md")
+
+    with open(gt_path) as f:
+        gt = json.load(f)
+
+    readme = ""
+    if os.path.exists(readme_path):
+        with open(readme_path) as f:
+            readme = f.read()
+
+    return {
+        "id": os.path.basename(bug_dir),
+        "title": gt.get("title", os.path.basename(bug_dir)),
+        "description": gt.get("description", ""),
+        "severity": gt.get("severity", "UNKNOWN"),
+        "category": gt.get("category", "unknown"),
+        "expected_findings": gt.get("findings", []),
+        "pre_diff": os.path.join(bug_dir, "pre.diff"),
+        "post_diff": os.path.join(bug_dir, "post.diff"),
+        "repo_path": os.path.join(bug_dir, "repo"),
+        "readme": readme,
+    }
+
+
+def _file_matches(exp_file: str, found_file: str) -> bool:
+    """Deterministic file match on basename. Empty either side = pass-through."""
+    if not exp_file or not found_file:
+        return True
+    return os.path.basename(exp_file) == os.path.basename(found_file)
+
+
+def _class_matches(exp: dict, fnd: dict) -> bool:
+    """Deterministic class match: both sides carry `classes` and they intersect."""
+    exp_classes = exp.get("classes") or []
+    fnd_classes = fnd.get("classes") or []
+    if not exp_classes or not fnd_classes:
+        return False  # not class-matchable — caller falls back to keywords
+    return bool(set(exp_classes) & set(fnd_classes))
+
+
+def score_run(bug: dict, run_info: dict) -> dict:
+    """Score a run against ground truth.
+
+    Matching is two-tier and deterministic:
+      Tier 1 (preferred): class-based — ground truth entries carry `classes`
+      (CWE/HG ids) and findings carry `classes`; a match requires a non-empty
+      class intersection AND a basename file match. No thresholds, no fuzzy
+      scores. Identical inputs always produce identical results.
+      Tier 2 (legacy fallback): keyword overlap for old results that predate
+      the classes field. Threshold fixed at 0.3, severity boost 1.3 — kept ONLY
+      for backward compatibility; new ground truth must carry classes.
+    """
+    expected = bug.get("expected_findings", [])
+    found = run_info.get("findings", [])
+
+    matched_expected = set()
+    matched_found = set()
+    matches = []
+    match_modes = {"class": 0, "keyword": 0}
+
+    for i, exp in enumerate(expected):
+        # ---- Tier 1: deterministic class match ----
+        if exp.get("classes"):
+            for j, fnd in enumerate(found):
+                if j in matched_found:
+                    continue
+                if _class_matches(exp, fnd) and _file_matches(exp.get("file", ""), fnd.get("file", "")):
+                    matched_expected.add(i)
+                    matched_found.add(j)
+                    match_modes["class"] += 1
+                    matches.append({
+                        "expected_id": f"EXP-{i+1}",
+                        "found_id": fnd.get("id", f"HG-{j+1}"),
+                        "mode": "class",
+                        "matched_classes": sorted(set(exp["classes"]) & set(fnd.get("classes") or [])),
+                        "expected_severity": exp.get("severity", "?"),
+                        "found_severity": fnd.get("severity", "?"),
+                        "expected_file": exp.get("file", "?"),
+                        "found_file": fnd.get("file", "?"),
+                    })
+                    break
+            continue  # class-carrying entries never fall through to keywords
+
+        # ---- Tier 2: legacy keyword fallback ----
+        exp_text = f"{exp.get('file','')} {exp.get('trigger','')} {exp.get('description','')}".lower()
+        best_score = 0
+        best_j = -1
+
+        for j, fnd in enumerate(found):
+            if j in matched_found:
+                continue
+            fnd_text = f"{fnd.get('file','')} {fnd.get('trigger_condition','')} {fnd.get('consequence','')}".lower()
+
+            # Simple keyword overlap score
+            exp_words = set(exp_text.split())
+            fnd_words = set(fnd_text.split())
+            if not exp_words or not fnd_words:
+                continue
+            overlap = len(exp_words & fnd_words) / min(len(exp_words), len(fnd_words))
+
+            # Boost: same severity
+            if exp.get("severity") == fnd.get("severity"):
+                overlap *= 1.3
+
+            if overlap > best_score:
+                best_score = overlap
+                best_j = j
+
+        if best_score > 0.3 and best_j >= 0:  # threshold for a "match"
+            matched_expected.add(i)
+            matched_found.add(best_j)
+            matched_fnd = found[best_j]
+            match_modes["keyword"] += 1
+            matches.append({
+                "expected_id": f"EXP-{i+1}",
+                "found_id": matched_fnd.get("id", f"HG-{best_j+1}"),
+                "mode": "keyword",
+                "score": round(best_score, 2),
+                "expected_severity": exp.get("severity", "?"),
+                "found_severity": matched_fnd.get("severity", "?"),
+                "expected_file": exp.get("file", "?"),
+                "found_file": matched_fnd.get("file", "?"),
+            })
+
+    total_expected = len(expected)
+    total_found = len(found)
+    true_positives = len(matched_expected)
+    false_negatives = total_expected - true_positives
+    false_positives = total_found - len(matched_found)
+
+    precision = round(true_positives / total_found * 100, 1) if total_found > 0 else 0
+    recall = round(true_positives / total_expected * 100, 1) if total_expected > 0 else 0
+    f1 = round(2 * precision * recall / (precision + recall), 1) if (precision + recall) > 0 else 0
+
+    # Severity accuracy
+    severity_matches = 0
+    for m in matches:
+        if m["expected_severity"] == m["found_severity"]:
+            severity_matches += 1
+    severity_accuracy = round(severity_matches / len(matches) * 100, 1) if matches else 0
+
+    return {
+        "bug_id": bug["id"],
+        "bug_title": bug["title"],
+        "bug_severity": bug["severity"],
+        "bug_category": bug["category"],
+        "expected_findings": total_expected,
+        "found_findings": total_found,
+        "true_positives": true_positives,
+        "false_negatives": false_negatives,
+        "false_positives": false_positives,
+        "precision_pct": precision,
+        "recall_pct": recall,
+        "f1_score": f1,
+        "severity_accuracy_pct": severity_accuracy,
+        "matches": matches,
+        "match_modes": match_modes,
+        # A result file without an explicit run_ok flag is treated as a
+        # completed run (backward compat) UNLESS it carries an error key —
+        # a crashed run must never be scored as OK.
+        "run_ok": bool(run_info.get("run_ok", not run_info.get("error"))),
+        "run_error": run_info.get("error"),
+    }
+
+
+def generate_report(scores: list, output_path: str):
+    """Generate HTML benchmark report."""
+    total_bugs = len(scores)
+    ok_runs = [s for s in scores if s["run_ok"]]
+    avg_precision = round(sum(s["precision_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0
+    avg_recall = round(sum(s["recall_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0
+    avg_f1 = round(sum(s["f1_score"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0
+    avg_severity = round(sum(s["severity_accuracy_pct"] for s in ok_runs) / len(ok_runs), 1) if ok_runs else 0
+
+    total_fp = sum(s["false_positives"] for s in ok_runs)
+    total_fn = sum(s["false_negatives"] for s in ok_runs)
+
+    html = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>HermaGuard Benchmark — {datetime.now().strftime('%d/%m/%Y')}</title>
+<style>
+  :root {{ color-scheme: dark; }}
+  * {{ margin:0; padding:0; box-sizing:border-box; }}
+  body {{ background:#11100f; color:#f5f5f4; font-family:system-ui,sans-serif; line-height:1.6; padding:2rem; }}
+  h1 {{ color:#fbbf24; font-size:1.5rem; margin-bottom:0.5rem; }}
+  .subtitle {{ color:#a8a29e; font-size:0.9rem; margin-bottom:2rem; }}
+  .summary {{ display:flex; gap:1.5rem; margin:1.5rem 0; flex-wrap:wrap; }}
+  .stat {{ background:#1c1a18; border:1px solid #34302c; border-radius:6px; padding:1rem 1.25rem; }}
+  .stat-value {{ font-size:1.5rem; font-weight:700; color:#fbbf24; }}
+  .stat-label {{ font-size:0.8rem; color:#a8a29e; }}
+  .good {{ color:#22c55e; }} .warn {{ color:#fbbf24; }} .bad {{ color:#ef4444; }}
+  table {{ width:100%; border-collapse:collapse; margin:1.5rem 0; }}
+  th, td {{ padding:0.6rem 0.8rem; text-align:left; border-bottom:1px solid #34302c; }}
+  th {{ color:#a8a29e; font-size:0.85rem; font-weight:600; }}
+  td {{ font-size:0.9rem; }}
+  .bug-title {{ max-width:300px; }}
+  .match {{ color:#22c55e; }} .miss {{ color:#ef4444; }}
+  .error-row {{ background:#3b1111; }}
+  h2 {{ color:#fbbf24; margin:2rem 0 1rem; padding-bottom:0.5rem; border-bottom:1px solid #34302c; }}
+  .footer {{ margin-top:3rem; padding-top:1rem; border-top:1px solid #34302c; color:#78716c; font-size:0.8rem; }}
+</style>
+</head>
+<body>
+
+<h1>HermaGuard Benchmark</h1>
+<div class="subtitle">{datetime.now().strftime('%d/%m/%Y %H:%M')} · {total_bugs} bugs · v{VERSION}</div>
+
+<div class="summary">
+  <div class="stat"><div class="stat-value">{avg_f1}<span style="font-size:0.8rem">%</span></div><div class="stat-label">Avg F1 Score</div></div>
+  <div class="stat"><div class="stat-value">{avg_recall}<span style="font-size:0.8rem">%</span></div><div class="stat-label">Avg Recall</div></div>
+  <div class="stat"><div class="stat-value">{avg_precision}<span style="font-size:0.8rem">%</span></div><div class="stat-label">Avg Precision</div></div>
+  <div class="stat"><div class="stat-value">{avg_severity}<span style="font-size:0.8rem">%</span></div><div class="stat-label">Severity Accuracy</div></div>
+  <div class="stat"><div class="stat-value">{total_fp}</div><div class="stat-label">False Positives</div></div>
+  <div class="stat"><div class="stat-value">{total_fn}</div><div class="stat-label">False Negatives</div></div>
+</div>
+
+<h2>Results by Bug</h2>
+<table>
+<tr><th>Bug</th><th>Severity</th><th>Expected</th><th>Found</th><th>TP</th><th>FP</th><th>FN</th><th>Precision</th><th>Recall</th><th>F1</th><th>Sev Acc</th></tr>
+"""
+
+    for s in scores:
+        if not s["run_ok"]:
+            html += f"""<tr class="error-row"><td class="bug-title">{s['bug_title'][:60]}</td>
+<td>-</td><td>-</td><td>-</td><td>-</td><td>-</td><td>-</td><td colspan="4">ERROR: {s.get('run_error','Unknown')[:100]}</td></tr>\n"""
+            continue
+
+        pcls = "good" if s["precision_pct"] >= 70 else ("warn" if s["precision_pct"] >= 40 else "bad")
+        rcls = "good" if s["recall_pct"] >= 70 else ("warn" if s["recall_pct"] >= 40 else "bad")
+        fcls = "good" if s["f1_score"] >= 70 else ("warn" if s["f1_score"] >= 40 else "bad")
+
+        html += f"""<tr>
+<td class="bug-title">{s['bug_title'][:60]}</td>
+<td>{s['bug_severity']}</td>
+<td>{s['expected_findings']}</td>
+<td>{s['found_findings']}</td>
+<td class="match">{s['true_positives']}</td>
+<td class="miss">{s['false_positives']}</td>
+<td class="miss">{s['false_negatives']}</td>
+<td class="{pcls}">{s['precision_pct']}%</td>
+<td class="{rcls}">{s['recall_pct']}%</td>
+<td class="{fcls}">{s['f1_score']}%</td>
+<td>{s['severity_accuracy_pct']}%</td>
+</tr>\n"""
+
+    html += "</table>\n"
+
+    # Detailed matches for each bug
+    html += "<h2>Match Details</h2>\n"
+    for s in scores:
+        if not s["run_ok"]:
+            continue
+        html += f"<h3>{s['bug_title'][:80]} ({s['bug_severity']})</h3>\n"
+        if s["matches"]:
+            html += "<table><tr><th>Expected</th><th>Found</th><th>Sev Match</th><th>Score</th></tr>\n"
+            for m in s["matches"]:
+                sev_match = "✓" if m["expected_severity"] == m["found_severity"] else "✗"
+                html += f"""<tr><td>{m['expected_file']} ({m['expected_severity']})</td>
+<td>{m['found_file'][:50]} ({m['found_severity']})</td><td>{sev_match}</td><td>{m['score']}</td></tr>\n"""
+            html += "</table>\n"
+        else:
+            html += "<p class='miss'>No matches — all findings missed or all expected bugs not found.</p>\n"
+
+        if s.get("matches"):
+            # Unmatched expected (FN)
+            matched_exp_ids = {m["expected_id"] for m in s["matches"]}
+            unmatched_exp = s.get("expected_findings_count", s["expected_findings"]) - len(matched_exp_ids)
+            if unmatched_exp > 0:
+                html += f"<p class='miss'>{unmatched_exp} expected bug{'s' if unmatched_exp > 1 else ''} not found (false negative).</p>\n"
+
+    html += f"""<div class="footer">
+HermaGuard Benchmark v{VERSION} · Generated {datetime.now().isoformat()}
+</div>
+</body></html>"""
+
+    with open(output_path, "w") as f:
+        f.write(html)
+
+    print(f"Report written to {output_path}")
+    return output_path
+
+
+def main():
+    parser = argparse.ArgumentParser(description="HermaGuard Benchmark — score JSON output against ground truth")
+    parser.add_argument("--bugs", default="bugs/", help="Path to bugs directory")
+    parser.add_argument("--bug", help="Score a single bug by path")
+    parser.add_argument("--results", default="results/", help="Directory containing per-bug JSON result files")
+    parser.add_argument("--result", help="Single result JSON file (use with --bug)")
+    parser.add_argument("--output", default="report.html", help="HTML report output path")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}")
+
+    args = parser.parse_args()
+
+    if args.bug and args.result:
+        # Single bug mode
+        bug = load_bug(args.bug)
+        with open(args.result) as f:
+            run_info = json.load(f)
+        s = score_run(bug, run_info)
+        status = "✓" if s["run_ok"] else "✗"
+        print(f"  {status} {bug['title'][:60]} — P:{s['precision_pct']}% R:{s['recall_pct']}% F1:{s['f1_score']}%")
+        return
+
+    bugs = discover_bugs(args.bugs)
+    if not bugs:
+        print(f"No bugs found in {args.bugs}")
+        sys.exit(1)
+
+    print(f"Found {len(bugs)} bug(s)")
+
+    scores = []
+    for bug_dir in bugs:
+        bug = load_bug(bug_dir)
+        result_path = os.path.join(args.results, f"{bug['id']}.json")
+
+        if os.path.exists(result_path):
+            with open(result_path) as f:
+                run_info = json.load(f)
+            s = score_run(bug, run_info)
+            scores.append(s)
+            status = "✓" if s["run_ok"] else "✗"
+            print(f"  {status} {bug['title'][:60]} — P:{s['precision_pct']}% R:{s['recall_pct']}% F1:{s['f1_score']}%")
+        else:
+            scores.append({
+                "bug_id": bug["id"], "bug_title": bug["title"],
+                "bug_severity": bug["severity"], "bug_category": bug["category"],
+                "run_ok": False, "run_error": f"No results file at {result_path}",
+                "precision_pct": 0, "recall_pct": 0, "f1_score": 0,
+                "expected_findings": len(bug["expected_findings"]), "found_findings": 0,
+                "true_positives": 0, "false_positives": 0, "false_negatives": len(bug["expected_findings"]),
+                "severity_accuracy_pct": 0, "matches": [],
+            })
+
+    # Generate report. --output is taken literally (path doubling bug fixed);
+    # create the parent dir so an explicit path like results/report.html works.
+    report_path = args.output
+    parent = os.path.dirname(report_path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    generate_report(scores, report_path)
+
+    ok = [s for s in scores if s["run_ok"]]
+    if ok:
+        avg_f1 = round(sum(s["f1_score"] for s in ok) / len(ok), 1)
+        print(f"\nOverall: {len(ok)}/{len(scores)} runs OK, avg F1: {avg_f1}%")
+    else:
+        print(f"\nAll runs lack result files. Run HermaGuard via your harness first, then score.")
+        print(f"Example: run '/hermaguard --json' under any harness, save output to results/<bug-id>.json")
+
+    print(f"Report: {report_path}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,8 @@
+import sqlite3
+
+def get_user_by_name(username: str):
+    conn = sqlite3.connect("users.db")
+    cur = conn.cursor()
+    query = f"SELECT * FROM users WHERE name = '{username}'"
+    cur.execute(query)
+    return cur.fetchone()
@@ -0,0 +1,9 @@
+{
+  "title": "SQL injection in get_user_by_name",
+  "description": "Username interpolated into SQL via f-string; attacker-controlled input reaches the query unsanitised.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {"file": "app.py", "trigger": "f-string SQL query username interpolation", "description": "SQL injection via unparameterised query", "severity": "CRITICAL", "classes": ["CWE-89"]}
+  ]
+}
@@ -0,0 +1,10 @@
+--- a/app.py
+++ b/app.py
+@@ -2,6 +2,6 @@ import sqlite3
+ def get_user_by_name(username: str):
+     conn = sqlite3.connect("users.db")
+     cur = conn.cursor()
+-    cur.execute("SELECT * FROM users WHERE name = ?", (username,))
+    query = f"SELECT * FROM users WHERE name = '{username}'"
+    cur.execute(query)
+     return cur.fetchone()
@@ -0,0 +1,5 @@
+import yaml
+
+def load_config(path: str) -> dict:
+    with open(path) as f:
+        return yaml.load(f)
@@ -0,0 +1,9 @@
+{
+  "title": "Unsafe YAML deserialisation",
+  "description": "yaml.load() without SafeLoader allows arbitrary object construction — RCE via crafted YAML.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {"file": "app.py", "trigger": "yaml.load without SafeLoader", "description": "Arbitrary code execution via unsafe deserialisation", "severity": "CRITICAL", "classes": ["CWE-502"]}
+  ]
+}
@@ -0,0 +1,7 @@
+--- a/app.py
+++ b/app.py
+@@ -2,4 +2,4 @@ import yaml
+ def load_config(path: str) -> dict:
+     with open(path) as f:
+-        return yaml.safe_load(f)
+        return yaml.load(f)
@@ -0,0 +1,4 @@
+def get_user_email(cur, user_id: int) -> str:
+    cur.execute("SELECT email FROM users WHERE id = ?", (user_id,))
+    row = cur.fetchone()
+    return row[0]
@@ -0,0 +1,9 @@
+{
+  "title": "Missing null check on fetchone result",
+  "description": "fetchone() returns None when no row matches; indexing row[0] then raises TypeError, crashing the caller.",
+  "severity": "MEDIUM",
+  "category": "logic",
+  "findings": [
+    {"file": "app.py", "trigger": "cursor fetchone row indexed without null check", "description": "TypeError crash when user_id not found", "severity": "MEDIUM", "classes": ["CWE-476", "CWE-754"]}
+  ]
+}
@@ -0,0 +1,8 @@
+--- a/app.py
+++ b/app.py
+@@ -2,4 +2,3 @@ def get_user_email(cur, user_id: int) -> str:
+     cur.execute("SELECT email FROM users WHERE id = ?", (user_id,))
+     row = cur.fetchone()
+-    if row is None:
+-        return None
+     return row[0]
@@ -0,0 +1,4 @@
+import subprocess
+
+def ping_host(host: str) -> bytes:
+    return subprocess.check_output(f"ping -c 1 {host}", shell=True)
@@ -0,0 +1,9 @@
+{
+  "title": "Shell command injection via ping utility",
+  "description": "User-supplied host is concatenated into a shell command; shell metacharacters execute arbitrary commands.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {"file": "app.py", "trigger": "user host concatenated into shell command", "description": "OS command injection via shell=True with unescaped input", "severity": "CRITICAL", "classes": ["CWE-78"]}
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.py
+++ b/app.py
+@@ -2,2 +2,2 @@ import subprocess
+ def ping_host(host: str) -> bytes:
+-    return subprocess.check_output(["ping", "-c", "1", host])
+    return subprocess.check_output(f"ping -c 1 {host}", shell=True)
@@ -0,0 +1,3 @@
+def read_doc(name: str) -> str:
+    with open("/var/docs/" + name) as f:
+        return f.read()
@@ -0,0 +1,9 @@
+{
+  "title": "Path traversal in file download",
+  "description": "User-supplied filename is joined without validation; ../ sequences escape the intended directory.",
+  "severity": "HIGH",
+  "category": "security",
+  "findings": [
+    {"file": "app.py", "trigger": "filename joined into path without traversal check", "description": "Arbitrary file read via ../ traversal", "severity": "HIGH", "classes": ["CWE-22"]}
+  ]
+}
@@ -0,0 +1,8 @@
+--- a/app.py
+++ b/app.py
+@@ -1,3 +1,3 @@
+ def read_doc(name: str) -> str:
+-    safe = os.path.basename(name)
+-    with open(os.path.join("/var/docs", safe)) as f:
+    with open("/var/docs/" + name) as f:
+         return f.read()
@@ -0,0 +1,11 @@
+# Deliberately broken sample: a live-looking API key committed to source.
+# NOTE: the value is an obviously-fake placeholder (marker, not a real key
+# format) so secret scanners do not flag the corpus; the CWE-798 pattern
+# (credential hardcoded in source) is what the benchmark tests for.
+PAYMENT_KEY = "EXAMPLE-KEY-DO-NOT-USE-0123456789"
+
+def charge(card_token: str, amount: int):
+    import requests
+    return requests.post("https://api.example-payments.invalid/v1/charges",
+                         headers={"Authorization": f"Bearer {PAYMENT_KEY}"},
+                         data={"card": card_token, "amount": amount})
@@ -0,0 +1,17 @@
+{
+  "title": "Hard-coded API key committed to source",
+  "description": "Live API key embedded in source; anyone with repo access can use the account.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "live API key assigned to module constant",
+      "description": "Credential theft and unauthorized charges from public repo",
+      "severity": "CRITICAL",
+      "classes": [
+        "CWE-798"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,4 @@
+--- a/app.py
+++ b/app.py
+@@ -0,0 +1 @@
+ (new file — the bug is introduced by this change)
@@ -0,0 +1,5 @@
+def delete_user(session_user, target_id: int):
+    # authn done upstream, but no role check
+    db.execute("DELETE FROM users WHERE id = ?", (target_id,))
+    db.commit()
+    return {"ok": True}
@@ -0,0 +1,18 @@
+{
+  "title": "Admin endpoint without authorization check",
+  "description": "delete_user performs no permission check; any authenticated user can delete anyone.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "admin action without role/permission check",
+      "description": "Privilege escalation \u2014 any user deletes any account",
+      "severity": "CRITICAL",
+      "classes": [
+        "CWE-862",
+        "CWE-269"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.py
+++ b/app.py
+ def delete_user(session_user, target_id: int):
+-    if session_user.get("role") != "admin":
+-        raise PermissionError("admin only")
+     # authn done upstream, but no role check
@@ -0,0 +1,3 @@
+def order_total(price_cents: int, quantity: int) -> int:
+    # no upper bound on quantity
+    return price_cents * quantity
@@ -0,0 +1,18 @@
+{
+  "title": "Unvalidated quantity causes integer overflow",
+  "description": "quantity from request body multiplies price without bounds; extreme values overflow.",
+  "severity": "MEDIUM",
+  "category": "logic",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "quantity multiplied without upper-bound check",
+      "description": "Overflow/wraparound corrupts totals on extreme input",
+      "severity": "MEDIUM",
+      "classes": [
+        "CWE-190",
+        "CWE-20"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.py
+++ b/app.py
+ def order_total(price_cents: int, quantity: int) -> int:
+-    if quantity > 10_000:
+-        raise ValueError("quantity too large")
+     # no upper bound on quantity
@@ -0,0 +1,5 @@
+import requests
+
+def send_webhook(url: str, payload: dict):
+    # user-supplied URL fetched without validation
+    requests.post(url, json=payload, timeout=5)
@@ -0,0 +1,18 @@
+{
+  "title": "SSRF via user-controlled webhook URL",
+  "description": "Webhook URL comes from the request and is fetched server-side without allowlist.",
+  "severity": "HIGH",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "user-supplied URL fetched server-side without allowlist",
+      "description": "SSRF \u2014 internal network probing and metadata endpoint access",
+      "severity": "HIGH",
+      "classes": [
+        "CWE-918",
+        "CWE-20"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,8 @@
+--- a/app.py
+++ b/app.py
+ import requests
+ 
+ def send_webhook(url: str, payload: dict):
+-    if not url.startswith("https://hooks.example.com/"):
+-        raise ValueError("webhook host not allowed")
+     # user-supplied URL fetched without validation
@@ -0,0 +1,5 @@
+function renderResults(results, searchTerm) {
+  const box = document.getElementById("results");
+  box.innerHTML = `<h2>Results for ${searchTerm}</h2>` +
+    results.map(r => `<li>${r.name}</li>`).join("");
+}
@@ -0,0 +1,17 @@
+{
+  "title": "Reflected XSS via innerHTML assignment",
+  "description": "Search term inserted into the DOM via innerHTML without escaping.",
+  "severity": "HIGH",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.js",
+      "trigger": "user input interpolated into innerHTML",
+      "description": "Script injection in victim browsers via crafted search term",
+      "severity": "HIGH",
+      "classes": [
+        "CWE-79"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,4 @@
+--- a/app.js
+++ b/app.js
+@@ -0,0 +1 @@
+ (new file — the bug is introduced by this change)
@@ -0,0 +1,6 @@
+def withdraw(conn, user_id: int, amount: int):
+    row = conn.execute("SELECT balance FROM accounts WHERE id = ?", (user_id,)).fetchone()
+    if row[0] < amount:
+        raise ValueError("insufficient funds")
+    conn.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", (amount, user_id))
+    conn.commit()
@@ -0,0 +1,17 @@
+{
+  "title": "TOCTOU race on balance check",
+  "description": "Check-then-deduct without lock or atomic update; concurrent requests double-spend.",
+  "severity": "HIGH",
+  "category": "concurrency",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "balance check and deduct are separate non-atomic statements",
+      "description": "Double-spend under concurrent requests",
+      "severity": "HIGH",
+      "classes": [
+        "CWE-362"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,11 @@
+--- a/app.py
+++ b/app.py
+ def withdraw(conn, user_id: int, amount: int):
+     row = conn.execute("SELECT balance FROM accounts WHERE id = ?", (user_id,)).fetchone()
+-    # single atomic guarded update
+-    cur = conn.execute("UPDATE accounts SET balance = balance - ? WHERE id = ? AND balance >= ?", (amount, user_id, amount))
+-    if cur.rowcount == 0:
+-        raise ValueError("insufficient funds")
+-    conn.commit()
+-    return
+     if row[0] < amount:
@@ -0,0 +1,5 @@
+async function loadProfile(userId) {
+  const res = await fetch(`/api/users/${userId}`);
+  const data = await res.json();
+  return data;
+}
@@ -0,0 +1,18 @@
+{
+  "title": "Unhandled promise rejection",
+  "description": "fetch promise has no catch; network failure leaves UI in loading state forever.",
+  "severity": "MEDIUM",
+  "category": "async",
+  "findings": [
+    {
+      "file": "app.js",
+      "trigger": "await fetch chain with no try/catch or .catch",
+      "description": "Unhandled rejection \u2014 loading state never resolves on failure",
+      "severity": "MEDIUM",
+      "classes": [
+        "HG-ASYNC",
+        "CWE-754"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,11 @@
+--- a/app.js
+++ b/app.js
+ async function loadProfile(userId) {
+-  try {
+     const res = await fetch(`/api/users/${userId}`);
+     const data = await res.json();
+     return data;
+-  } catch (e) {
+-    showError(e);
+-  }
+ }
@@ -0,0 +1,6 @@
+def process_payment(order):
+    try:
+        gateway.charge(order)
+    except Exception:
+        pass
+    return {"status": "paid"}
@@ -0,0 +1,18 @@
+{
+  "title": "Swallowed exception in payment processing",
+  "description": "Bare except: pass hides every failure; function returns fake success.",
+  "severity": "HIGH",
+  "category": "error-handling",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "bare except pass, returns success regardless",
+      "description": "Fake success masks failed charges \u2014 silent data/money loss",
+      "severity": "HIGH",
+      "classes": [
+        "HG-DEGENERATE",
+        "CWE-754"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,10 @@
+--- a/app.py
+++ b/app.py
+ def process_payment(order):
+     try:
+         gateway.charge(order)
+     except Exception:
+-        log.exception("charge failed")
+-        return {"status": "failed"}
+        pass
+     return {"status": "paid"}
@@ -0,0 +1,4 @@
+def transfer(conn, from_id, to_id, amount):
+    conn.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", (amount, from_id))
+    conn.execute("UPDATE accounts SET balance = balance + ? WHERE id = ?", (amount, to_id))
+    conn.commit()
@@ -0,0 +1,18 @@
+{
+  "title": "Non-atomic multi-step update",
+  "description": "Two-table update without transaction; crash between statements corrupts state.",
+  "severity": "HIGH",
+  "category": "data-integrity",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "two dependent updates without a wrapping transaction",
+      "description": "Crash mid-sequence loses money \u2014 debit without credit",
+      "severity": "HIGH",
+      "classes": [
+        "HG-PARTIAL-WRITE",
+        "CWE-362"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,7 @@
+--- a/app.py
+++ b/app.py
+ def transfer(conn, from_id, to_id, amount):
+-    with conn:
+         conn.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", (amount, from_id))
+         conn.execute("UPDATE accounts SET balance = balance + ? WHERE id = ?", (amount, to_id))
+-    conn.commit()
@@ -0,0 +1,4 @@
+def read_config(path):
+    f = open(path)
+    data = json.load(f)
+    return data
@@ -0,0 +1,18 @@
+{
+  "title": "Unclosed file handle on error path",
+  "description": "open() without context manager; exception path leaks the descriptor.",
+  "severity": "MEDIUM",
+  "category": "resources",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "open without with/close on all paths",
+      "description": "FD leak under repeated failures exhausts process limit",
+      "severity": "MEDIUM",
+      "classes": [
+        "CWE-404",
+        "CWE-754"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,7 @@
+--- a/app.py
+++ b/app.py
+ def read_config(path):
+-    with open(path) as f:
+-        return json.load(f)
+    f = open(path)
+     data = json.load(f)
@@ -0,0 +1,5 @@
+def handler(event):
+    try:
+        return do_work(event)
+    except Exception as e:
+        return {"statusCode": 500, "body": str(e)}
@@ -0,0 +1,17 @@
+{
+  "title": "Stack trace returned to client",
+  "description": "Bare handler returns exception text in the HTTP response body.",
+  "severity": "MEDIUM",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "exception text returned in response body",
+      "description": "Internal paths, queries, and versions leaked to clients",
+      "severity": "MEDIUM",
+      "classes": [
+        "CWE-200"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,4 @@
+--- a/app.py
+++ b/app.py
+@@ -0,0 +1 @@
+ (new file — the bug is introduced by this change)
@@ -0,0 +1,8 @@
+def parse(stream):
+    records = []
+    while True:
+        line = stream.readline()
+        if not line:
+            break
+        records.append(json.loads(line))
+    return records
@@ -0,0 +1,17 @@
+{
+  "title": "Unbounded loop over user input",
+  "description": "parse runs until end-of-stream with no cap on iterations or size.",
+  "severity": "MEDIUM",
+  "category": "resources",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "read loop without record-count or byte cap",
+      "description": "Memory exhaustion DoS from large uploads",
+      "severity": "MEDIUM",
+      "classes": [
+        "CWE-400"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,12 @@
+--- a/app.py
+++ b/app.py
+ def parse(stream):
+     records = []
+    MAX = 10_000
+     while True:
+         line = stream.readline()
+         if not line:
+             break
+        if len(records) >= MAX:
+            raise ValueError("too many records")
+         records.append(json.loads(line))
@@ -0,0 +1,6 @@
+import hashlib
+
+def verify(shared_secret: str, provided: str) -> bool:
+    a = hashlib.sha256(shared_secret.encode()).hexdigest()
+    b = hashlib.sha256(provided.encode()).hexdigest()
+    return a == b
@@ -0,0 +1,18 @@
+{
+  "title": "Password comparison not constant-time",
+  "description": "Plain == on password-derived token allows timing side channel.",
+  "severity": "MEDIUM",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "secret compared with plain string equality",
+      "description": "Timing side channel leaks secret one byte at a time",
+      "severity": "MEDIUM",
+      "classes": [
+        "CWE-287",
+        "CWE-200"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,10 @@
+--- a/app.py
+++ b/app.py
+ import hashlib
+import hmac
+ 
+ def verify(shared_secret: str, provided: str) -> bool:
+     a = hashlib.sha256(shared_secret.encode()).hexdigest()
+     b = hashlib.sha256(provided.encode()).hexdigest()
+-    return hmac.compare_digest(a, b)
+    return a == b
@@ -0,0 +1,4 @@
+import pickle
+
+def load_state(body: bytes):
+    return pickle.loads(body)
@@ -0,0 +1,18 @@
+{
+  "title": "Pickle load of user-supplied bytes",
+  "description": "pickle.loads on request body executes arbitrary code during deserialisation.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "pickle.loads on untrusted request bytes",
+      "description": "RCE \u2014 pickle deserialisation executes attacker code",
+      "severity": "CRITICAL",
+      "classes": [
+        "CWE-502",
+        "CWE-94"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,4 @@
+--- a/app.py
+++ b/app.py
+@@ -0,0 +1 @@
+ (new file — the bug is introduced by this change)
@@ -0,0 +1,4 @@
+def reset_password(request, user_id: int):
+    new_pw = request.json["password"]
+    db.execute("UPDATE users SET pw = ? WHERE id = ?", (hash(new_pw), user_id))
+    db.commit()
@@ -0,0 +1,18 @@
+{
+  "title": "State-changing route without authentication",
+  "description": "POST handler has no session check; anonymous requests reset any account.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "state-changing handler without session/auth check",
+      "description": "Account takeover by anonymous callers",
+      "severity": "CRITICAL",
+      "classes": [
+        "CWE-306",
+        "CWE-287"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.py
+++ b/app.py
+ def reset_password(request, user_id: int):
+-    if not request.session.get("user"):
+-        raise PermissionError("login required")
+     new_pw = request.json["password"]
@@ -0,0 +1,3 @@
+function canEdit(order) {
+  return order.status == 1;  // "1" also passes
+}
@@ -0,0 +1,17 @@
+{
+  "title": "Loose equality on status check",
+  "description": "JS == with string/number mix accepts wrong status values.",
+  "severity": "LOW",
+  "category": "logic",
+  "findings": [
+    {
+      "file": "app.js",
+      "trigger": "loose == between string and number",
+      "description": "Wrong-type status grants edit rights unexpectedly",
+      "severity": "LOW",
+      "classes": [
+        "CWE-20"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.js
+++ b/app.js
+ function canEdit(order) {
+-  return order.status === 1;
+  return order.status == 1;  // "1" also passes
+ }
@@ -0,0 +1,8 @@
+async function refresh() {
+  state.loading = true;
+  try {
+    state.data = await api.get("/data");
+  } catch (e) {
+    state.error = e.message;
+  }
+}
@@ -0,0 +1,18 @@
+{
+  "title": "Missing loading\u2192error UI transition",
+  "description": "Error path updates message but never clears the loading flag.",
+  "severity": "MEDIUM",
+  "category": "logic",
+  "findings": [
+    {
+      "file": "app.js",
+      "trigger": "catch path never resets loading flag",
+      "description": "Spinner stuck forever after a failed refresh",
+      "severity": "MEDIUM",
+      "classes": [
+        "HG-STATE",
+        "HG-ASYNC"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.js
+++ b/app.js
+   } catch (e) {
+     state.error = e.message;
+    state.loading = false;
+   }
@@ -0,0 +1,5 @@
+def change_email(request):
+    if request.method in ("GET", "POST"):
+        u = db.get_user(request.args["user"])
+        u.email = request.args["email"]
+        u.save()
@@ -0,0 +1,17 @@
+{
+  "title": "State change on GET without CSRF token",
+  "description": "Password change honors GET with no token; img tags can trigger it.",
+  "severity": "HIGH",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "state change accepted on GET without CSRF token",
+      "description": "CSRF \u2014 attacker sites change victim emails via img/link",
+      "severity": "HIGH",
+      "classes": [
+        "CWE-352"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+--- a/app.py
+++ b/app.py
+ def change_email(request):
+-    if request.method == "POST" and request.form.get("csrf_token") == session_csrf():
+    if request.method in ("GET", "POST"):
+         u = db.get_user(request.args["user"])
@@ -0,0 +1,2 @@
+def apply_filter(value, expr: str):
+    return eval(f"{value} {expr}")
@@ -0,0 +1,17 @@
+{
+  "title": "eval on template expression",
+  "description": "Template renderer evals user-supplied filter expressions.",
+  "severity": "CRITICAL",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "eval on user-controlled expression string",
+      "description": "Arbitrary code execution via crafted filter",
+      "severity": "CRITICAL",
+      "classes": [
+        "CWE-94"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,4 @@
+--- a/app.py
+++ b/app.py
+@@ -0,0 +1 @@
+ (new file — the bug is introduced by this change)
@@ -0,0 +1,6 @@
+import os
+
+def save_upload(file_storage):
+    dest = os.path.join("/srv/uploads", file_storage.filename)
+    file_storage.save(dest)
+    return dest
@@ -0,0 +1,18 @@
+{
+  "title": "Unrestricted upload accepts any type",
+  "description": "Upload saves whatever the client sends under the original extension.",
+  "severity": "HIGH",
+  "category": "security",
+  "findings": [
+    {
+      "file": "app.py",
+      "trigger": "upload saved without extension/content-type validation",
+      "description": "Webshell upload via .php/.jsp under web root",
+      "severity": "HIGH",
+      "classes": [
+        "CWE-434",
+        "CWE-22"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,9 @@
+--- a/app.py
+++ b/app.py
+ import os
+ 
+ def save_upload(file_storage):
+-    ext = os.path.splitext(file_storage.filename)[1].lower()
+-    if ext not in (".png", ".jpg", ".pdf"):
+-        raise ValueError("file type not allowed")
+     dest = os.path.join("/srv/uploads", file_storage.filename)