evandrodevbr
c17192c9fe
Copia local (nao fork, nao submodule) de safedeps, sca-audit, secrets-scan, code-review-security, api-security-review, web-security-review, cve-triage, patch-prioritization, dependency-scanning e hermaguard -- commit exato de origem e licenca de cada uma em skills/NOTICE.md. Elimina o passo de clone externo pro pilar de seguranca do wf-security-review: git clone deste repo ja traz tudo funcional. Corrige tambem o comando de instalacao pro Claude Code nos dois READMEs e no docs/SKILLS.md: Claude Code exige um diretorio por skill (skills/<nome>/SKILL.md), nao aceita os .md soltos que o comando antigo copiava -- confirmado testando a instalacao real.
389 lines
17 KiB
Python
389 lines
17 KiB
Python
"""
|
|
Hermaguard Coverage — Test Suite
|
|
|
|
Tests import and call the real tool. No placebo assertions.
|
|
Covers: registry integrity, ledger build, report validation (pass/fail),
|
|
language rules, CLI round-trip.
|
|
"""
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
|
|
|
|
def _load_module(name, relpath):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT / relpath)
|
|
if spec is None or spec.loader is None:
|
|
raise ImportError(f"Could not load {relpath}")
|
|
mod = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = mod # required for dataclasses with from __future__ annotations
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
COV = _load_module("hermaguard_coverage", "tools/hermaguard-coverage/hermaguard-coverage.py")
|
|
|
|
|
|
def _full_coverage(ledger, finding_class=None):
|
|
"""Build a complete coverage array for the given ledger."""
|
|
cov = []
|
|
for c in ledger["classes"]:
|
|
if not c["applicable"]:
|
|
cov.append({"class_id": c["id"], "status": "not_applicable", "reason": c.get("reason", "")})
|
|
elif c["id"] == finding_class:
|
|
cov.append({"class_id": c["id"], "status": "finding"})
|
|
else:
|
|
cov.append({"class_id": c["id"], "status": "clean"})
|
|
return cov
|
|
|
|
|
|
class TestRegistryIntegrity(unittest.TestCase):
|
|
def test_self_check_passes(self):
|
|
self.assertEqual(COV.self_check(), [])
|
|
|
|
def test_registry_size(self):
|
|
# 25 CWE Top-25 + 3 CWE extras + 4 Hermaguard-native
|
|
self.assertEqual(len(COV.CLASSES), 32)
|
|
|
|
def test_top25_ids_exact(self):
|
|
ranked = [c for c in COV.CLASSES if c.get("rank") and 1 <= c["rank"] <= 25]
|
|
self.assertEqual(len(ranked), 25)
|
|
expected = {"CWE-79","CWE-787","CWE-89","CWE-352","CWE-22","CWE-125","CWE-78",
|
|
"CWE-416","CWE-862","CWE-434","CWE-94","CWE-20","CWE-77","CWE-287",
|
|
"CWE-269","CWE-502","CWE-200","CWE-863","CWE-918","CWE-119","CWE-476",
|
|
"CWE-798","CWE-190","CWE-400","CWE-306"}
|
|
self.assertEqual({c["id"] for c in ranked}, expected)
|
|
|
|
def test_top25_ranks_unique(self):
|
|
ranks = [c["rank"] for c in COV.CLASSES if c.get("rank")]
|
|
self.assertEqual(sorted(ranks), list(range(1, 26)))
|
|
|
|
def test_registry_mutation_detected(self):
|
|
# Tamper detection: adding a duplicate id must fail self-check
|
|
COV.CLASSES.append({"id": "CWE-89", "name": "dup", "status": "always"})
|
|
try:
|
|
errs = COV.self_check()
|
|
self.assertTrue(any("duplicate" in e for e in errs))
|
|
finally:
|
|
COV.CLASSES.pop()
|
|
|
|
|
|
class TestLedgerBuild(unittest.TestCase):
|
|
def test_python_only(self):
|
|
led = COV.build_ledger(["src/app.py"])
|
|
na = {c["id"] for c in led["classes"] if not c["applicable"]}
|
|
# memory-unsafe classes excluded for Python
|
|
self.assertIn("CWE-787", na)
|
|
self.assertIn("CWE-416", na)
|
|
applicable = {c["id"] for c in led["classes"] if c["applicable"]}
|
|
self.assertIn("CWE-89", applicable)
|
|
self.assertIn("HG-ASYNC", applicable)
|
|
|
|
def test_c_files_get_memory_classes(self):
|
|
led = COV.build_ledger(["driver.c"])
|
|
applicable = {c["id"] for c in led["classes"] if c["applicable"]}
|
|
self.assertIn("CWE-787", applicable)
|
|
self.assertIn("CWE-125", applicable)
|
|
self.assertIn("CWE-416", applicable)
|
|
|
|
def test_rust_gets_memory_classes_as_hint(self):
|
|
led = COV.build_ledger(["main.rs"])
|
|
by_id = {c["id"]: c for c in led["classes"]}
|
|
self.assertTrue(by_id["CWE-787"]["applicable"])
|
|
self.assertEqual(by_id["CWE-787"].get("hint"), "Rust: applies via unsafe blocks")
|
|
|
|
def test_mixed_languages_union(self):
|
|
led = COV.build_ledger(["a.py", "b.ts", "c.go"])
|
|
applicable = {c["id"] for c in led["classes"] if c["applicable"]}
|
|
# union: TS brings nothing memory-unsafe; all still N/A
|
|
self.assertNotIn("CWE-787", applicable)
|
|
|
|
def test_files_sorted_in_ledger(self):
|
|
led = COV.build_ledger(["b.py", "a.py"])
|
|
self.assertEqual(led["files"], ["a.py", "b.py"])
|
|
|
|
|
|
class TestReportValidation(unittest.TestCase):
|
|
def setUp(self):
|
|
self.ledger = COV.build_ledger(["src/app.py", "src/handler.ts"])
|
|
|
|
def test_full_coverage_passes(self):
|
|
report = {
|
|
"coverage": _full_coverage(self.ledger, finding_class="CWE-89"),
|
|
"findings": [{"id": "HG-1", "classes": ["CWE-89"], "file": "app.py",
|
|
"severity": "CRITICAL", "trigger_condition": "f-string SQL",
|
|
"consequence": "injection"}],
|
|
}
|
|
result = COV.check_report(report, self.ledger)
|
|
self.assertTrue(result["ok"], result["errors"])
|
|
self.assertEqual(result["accounted"], result["applicable"])
|
|
|
|
def test_missing_coverage_entry_fails(self):
|
|
cov = [e for e in _full_coverage(self.ledger) if e["class_id"] != "CWE-400"]
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertIn("CWE-400", result["gaps"])
|
|
|
|
def test_no_coverage_array_fails(self):
|
|
result = COV.check_report({"findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertIn("no `coverage` array", result["errors"][0])
|
|
|
|
def test_finding_claim_without_backing_finding_fails(self):
|
|
cov = _full_coverage(self.ledger, finding_class="CWE-22")
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertTrue(any("no finding cites this class" in e for e in result["errors"]))
|
|
|
|
def test_duplicate_class_entry_fails(self):
|
|
cov = _full_coverage(self.ledger)
|
|
cov.append({"class_id": "CWE-89", "status": "clean"})
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertTrue(any("appears twice" in e for e in result["errors"]))
|
|
|
|
def test_invalid_status_fails(self):
|
|
cov = _full_coverage(self.ledger)
|
|
for e in cov:
|
|
if e["class_id"] == "HG-STATE":
|
|
e["status"] = "maybe"
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertTrue(any("invalid status" in e for e in result["errors"]))
|
|
|
|
def test_na_without_reason_fails(self):
|
|
cov = _full_coverage(self.ledger)
|
|
for e in cov:
|
|
if e["class_id"] == "CWE-89":
|
|
e["status"] = "not_applicable" # demote without reason
|
|
e.pop("reason", None)
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertTrue(any("without a reason" in e for e in result["errors"]))
|
|
|
|
def test_unknown_class_cited_fails(self):
|
|
cov = _full_coverage(self.ledger)
|
|
cov.append({"class_id": "CWE-9999", "status": "clean"})
|
|
result = COV.check_report({"coverage": cov, "findings": []}, self.ledger)
|
|
self.assertFalse(result["ok"])
|
|
self.assertTrue(any("unknown class" in e for e in result["errors"]))
|
|
|
|
|
|
class TestCLI(unittest.TestCase):
|
|
def test_cli_self_check(self):
|
|
r = subprocess.run(
|
|
[sys.executable, str(ROOT / "tools/hermaguard-coverage/hermaguard-coverage.py"), "--self-check"],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
self.assertIn("self-check OK", r.stdout)
|
|
|
|
def test_cli_round_trip(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
led_path = str(Path(d) / "ledger.json")
|
|
rep_ok = str(Path(d) / "ok.json")
|
|
rep_bad = str(Path(d) / "bad.json")
|
|
|
|
r = subprocess.run(
|
|
[sys.executable, str(ROOT / "tools/hermaguard-coverage/hermaguard-coverage.py"),
|
|
"--files", "a.py,b.ts", "--ledger", led_path],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
|
|
ledger = json.loads(Path(led_path).read_text())
|
|
ok_report = {"coverage": _full_coverage(ledger, "CWE-89"),
|
|
"findings": [{"id": "HG-1", "classes": ["CWE-89"]}]}
|
|
Path(rep_ok).write_text(json.dumps(ok_report))
|
|
bad_cov = [e for e in ok_report["coverage"] if e["class_id"] != "HG-PARTIAL-WRITE"]
|
|
Path(rep_bad).write_text(json.dumps({"coverage": bad_cov, "findings": []}))
|
|
|
|
tool = str(ROOT / "tools/hermaguard-coverage/hermaguard-coverage.py")
|
|
r_ok = subprocess.run([sys.executable, tool, "--check-report", rep_ok, "--ledger", led_path, "--quiet"],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(r_ok.returncode, 0, r_ok.stdout + r_ok.stderr)
|
|
r_bad = subprocess.run([sys.executable, tool, "--check-report", rep_bad, "--ledger", led_path, "--quiet"],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(r_bad.returncode, 1)
|
|
self.assertIn("HG-PARTIAL-WRITE", r_bad.stdout)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
|
|
def _load_grader():
|
|
return _load_module("hermaguard_grader", "tools/hermaguard-grader/hermaguard_grader.py")
|
|
|
|
|
|
class TestGrader(unittest.TestCase):
|
|
"""L0-L6 grader — real oracle behaviour, not just self-test passthrough."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.g = _load_grader()
|
|
|
|
def test_self_test_corpus_passes(self):
|
|
rc = self.g.self_test()
|
|
self.assertEqual(rc, 0)
|
|
|
|
def test_privilege_axis_gives_l5(self):
|
|
f = self.g.grade_finding({"id": "X", "severity": "HIGH",
|
|
"harm_axes": {"completed": True, "reversible": False,
|
|
"cross_scope": False, "privilege": True}})
|
|
self.assertEqual(f["L_level"], "L5")
|
|
|
|
def test_severity_fallback_never_undergrades(self):
|
|
for tier, floor in self.g.SEVERITY_FALLBACK_FLOOR.items():
|
|
f = self.g.grade_finding({"id": "X", "severity": tier})
|
|
self.assertGreaterEqual(self.g._LEVEL_RANK[f["L_level"]],
|
|
self.g._LEVEL_RANK[floor], msg=tier)
|
|
self.assertEqual(f["grading_method"], "severity-fallback")
|
|
|
|
def test_grading_is_additive(self):
|
|
original = {"id": "X", "severity": "LOW", "keep": "me"}
|
|
f = self.g.grade_finding(dict(original))
|
|
self.assertEqual(f["keep"], "me")
|
|
for k in original:
|
|
self.assertIn(k, f)
|
|
|
|
def test_escalation_chain_raises_peak_to_l6(self):
|
|
report = {"findings": [
|
|
{"id": "A", "severity": "HIGH", "harm_axes": {"completed": True, "reversible": True,
|
|
"cross_scope": True, "privilege": False}}, # L4
|
|
{"id": "B", "severity": "CRITICAL", "harm_axes": {"completed": True, "reversible": False,
|
|
"cross_scope": False, "privilege": True}}, # L5
|
|
]}
|
|
out = self.g.grade_report(report)
|
|
by_id = {f["id"]: f for f in out["findings"]}
|
|
self.assertEqual(by_id["B"]["L_level"], "L6")
|
|
self.assertTrue(out["severity_grading"]["escalation_chain_detected"])
|
|
|
|
def test_local_chain_not_raised(self):
|
|
report = {"findings": [
|
|
{"id": "A", "severity": "MEDIUM", "harm_axes": {"completed": True, "reversible": True,
|
|
"cross_scope": False, "privilege": False}}, # L2
|
|
{"id": "B", "severity": "HIGH", "harm_axes": {"completed": True, "reversible": False,
|
|
"cross_scope": False, "privilege": False}}, # L3
|
|
]}
|
|
out = self.g.grade_report(report)
|
|
self.assertFalse(out["severity_grading"]["escalation_chain_detected"])
|
|
by_id = {f["id"]: f for f in out["findings"]}
|
|
self.assertEqual(by_id["B"]["L_level"], "L3")
|
|
|
|
|
|
def _load_benchmark():
|
|
return _load_module("hermaguard_benchmark", "tools/hermaguard-benchmark/benchmark.py")
|
|
|
|
|
|
class TestBenchmarkDeterministicScoring(unittest.TestCase):
|
|
"""Class-based matching replaces fuzzy keyword overlap for new ground truth."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.bm = _load_benchmark()
|
|
cls.bug = {
|
|
"id": "999-test", "title": "t", "severity": "HIGH", "category": "security",
|
|
"expected_findings": [
|
|
{"file": "app.py", "trigger": "x", "description": "x", "severity": "HIGH", "classes": ["CWE-89"]},
|
|
],
|
|
}
|
|
|
|
def test_class_match_exact(self):
|
|
run = {"run_ok": True, "findings": [
|
|
{"id": "HG-1", "file": "app.py", "severity": "HIGH", "classes": ["CWE-89", "CWE-20"]},
|
|
]}
|
|
s = self.bm.score_run(self.bug, run)
|
|
self.assertEqual(s["true_positives"], 1)
|
|
self.assertEqual(s["false_positives"], 0)
|
|
self.assertEqual(s["match_modes"]["class"], 1)
|
|
|
|
def test_class_match_requires_file_match(self):
|
|
run = {"run_ok": True, "findings": [
|
|
{"id": "HG-1", "file": "other.py", "severity": "HIGH", "classes": ["CWE-89"]},
|
|
]}
|
|
s = self.bm.score_run(self.bug, run)
|
|
self.assertEqual(s["true_positives"], 0)
|
|
self.assertEqual(s["false_negatives"], 1)
|
|
|
|
def test_no_class_intersection_no_match(self):
|
|
run = {"run_ok": True, "findings": [
|
|
{"id": "HG-1", "file": "app.py", "severity": "HIGH", "classes": ["CWE-22"]},
|
|
]}
|
|
s = self.bm.score_run(self.bug, run)
|
|
self.assertEqual(s["true_positives"], 0)
|
|
|
|
def test_deterministic_identical_inputs(self):
|
|
run = {"run_ok": True, "findings": [
|
|
{"id": "HG-1", "file": "app.py", "severity": "HIGH", "classes": ["CWE-89"]},
|
|
{"id": "HG-2", "file": "app.py", "severity": "MEDIUM", "classes": ["CWE-22"]},
|
|
]}
|
|
r1 = self.bm.score_run(self.bug, run)
|
|
r2 = self.bm.score_run(self.bug, run)
|
|
self.assertEqual(r1["matches"], r2["matches"])
|
|
self.assertEqual((r1["precision_pct"], r1["recall_pct"]), (r2["precision_pct"], r2["recall_pct"]))
|
|
|
|
def test_legacy_keyword_fallback_still_works(self):
|
|
bug_legacy = {
|
|
"id": "998-legacy", "title": "t", "severity": "HIGH", "category": "security",
|
|
"expected_findings": [
|
|
{"file": "app.py", "trigger": "sql injection unparameterised query", "description": "sqli", "severity": "CRITICAL"},
|
|
],
|
|
}
|
|
run = {"run_ok": True, "findings": [
|
|
{"id": "HG-1", "file": "app.py", "severity": "CRITICAL",
|
|
"trigger_condition": "sql injection unparameterised query", "consequence": "data breach"},
|
|
]}
|
|
s = self.bm.score_run(bug_legacy, run)
|
|
self.assertEqual(s["true_positives"], 1)
|
|
self.assertEqual(s["match_modes"]["keyword"], 1)
|
|
|
|
def test_crashed_run_never_scored_ok(self):
|
|
# Regression: a result file with an error key must not score as OK
|
|
run = {"error": "agent crashed", "findings": []}
|
|
s = self.bm.score_run(self.bug, run)
|
|
self.assertFalse(s["run_ok"])
|
|
|
|
def test_missing_run_ok_flag_does_not_crash(self):
|
|
# Regression: result files predating the run_ok flag previously
|
|
# raised KeyError inside score_run, killing the whole benchmark.
|
|
run = {"findings": [
|
|
{"id": "HG-1", "file": "app.py", "severity": "HIGH", "classes": ["CWE-89"]},
|
|
]}
|
|
s = self.bm.score_run(self.bug, run)
|
|
self.assertTrue(s["run_ok"]) # defaults to completed
|
|
self.assertEqual(s["true_positives"], 1)
|
|
|
|
def test_real_corpus_scoring_perfect_and_zero(self):
|
|
import pathlib
|
|
bugs_dir = pathlib.Path("tools/hermaguard-benchmark/bugs")
|
|
# Perfect run against real ground truth -> 100/100/100
|
|
gt = json.loads((bugs_dir / "001-sql-injection/ground-truth.json").read_text())
|
|
bug = {"id": "001", "title": gt["title"], "severity": gt["severity"],
|
|
"category": gt["category"], "expected_findings": gt["findings"]}
|
|
exp = gt["findings"][0]
|
|
run = {"findings": [{"id": "HG-1", "file": exp["file"],
|
|
"severity": exp["severity"], "classes": exp["classes"]}]}
|
|
s = self.bm.score_run(bug, run)
|
|
self.assertEqual((s["precision_pct"], s["recall_pct"], s["f1_score"]), (100.0, 100.0, 100.0))
|
|
# Wrong-class run -> 0
|
|
run2 = {"findings": [{"id": "HG-1", "file": exp["file"],
|
|
"severity": exp["severity"], "classes": ["CWE-79"]}]}
|
|
s2 = self.bm.score_run(bug, run2)
|
|
self.assertEqual((s2["precision_pct"], s2["recall_pct"], s2["f1_score"]), (0.0, 0.0, 0.0))
|
|
|
|
def test_corpus_integrity(self):
|
|
import pathlib
|
|
bugs_dir = pathlib.Path("tools/hermaguard-benchmark/bugs")
|
|
dirs = sorted([p for p in bugs_dir.iterdir() if p.is_dir()])
|
|
self.assertEqual(len(dirs), 25, f"expected 25 bugs, found {len(dirs)}")
|
|
for d in dirs:
|
|
gt = json.loads((d / "ground-truth.json").read_text())
|
|
self.assertTrue(gt["findings"], f"{d.name}: no findings in ground truth")
|
|
for f in gt["findings"]:
|
|
self.assertIn("classes", f, f"{d.name}: ground truth missing classes")
|