feat: vendorizar as 10 skills do pilar de seguranca em skills/
Copia local (nao fork, nao submodule) de safedeps, sca-audit, secrets-scan, code-review-security, api-security-review, web-security-review, cve-triage, patch-prioritization, dependency-scanning e hermaguard -- commit exato de origem e licenca de cada uma em skills/NOTICE.md. Elimina o passo de clone externo pro pilar de seguranca do wf-security-review: git clone deste repo ja traz tudo funcional. Corrige tambem o comando de instalacao pro Claude Code nos dois READMEs e no docs/SKILLS.md: Claude Code exige um diretorio por skill (skills/<nome>/SKILL.md), nao aceita os .md soltos que o comando antigo copiava -- confirmado testando a instalacao real.
This commit is contained in:
evandrodevbr committed
1 parent
ee44fa0f7a
commit
c17192c9fe
130 files changed
+10533
-33
No files matched your search
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
Hermaguard Verify — Test Suite
|
||||
|
||||
Runs REAL PoCs in the REAL sandbox. No mocks of the verdict logic:
|
||||
every test executes actual subprocesses and asserts on actual evidence.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def _load_module(name, relpath):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / relpath)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"Could not load {relpath}")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
V = _load_module("hermaguard_verify", "tools/hermaguard-verify/hermaguard-verify.py")
|
||||
|
||||
|
||||
def _write(tmpdir, name, content):
|
||||
p = Path(tmpdir) / name
|
||||
p.write_text(content)
|
||||
return str(p)
|
||||
|
||||
|
||||
class TestRealPoCRuns(unittest.TestCase):
|
||||
"""Each test writes a genuine PoC + target and runs the real sandbox."""
|
||||
|
||||
def test_verified_on_demonstrated_crash(self):
|
||||
# Target has the classic null-deref bug (bug 003 from the corpus)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
target = _write(d, "app.py",
|
||||
'def get_user_email(cur, user_id):\n row = cur.fetchone()\n return row[0]\n')
|
||||
# Real PoC: a stub cursor that returns None, like a missing row
|
||||
poc = _write(d, "poc.py",
|
||||
"import app\n"
|
||||
"class Cur:\n"
|
||||
" def fetchone(self):\n"
|
||||
" return None\n"
|
||||
"app.get_user_email(Cur(), 123)\n")
|
||||
ev = V.run_poc(poc, target, expect_fail=True, expect_output="TypeError")
|
||||
self.assertEqual(ev["verdict"], "VERIFIED", ev)
|
||||
self.assertIn("TypeError", ev["stderr"] + ev["stdout"])
|
||||
|
||||
def test_refuted_when_no_bug(self):
|
||||
# Fixed target: no bug, PoC runs clean -> REFUTED (likely false positive)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
target = _write(d, "app.py",
|
||||
'def get_user_email(cur, user_id):\n row = cur.fetchone()\n'
|
||||
' return None if row is None else row[0]\n')
|
||||
poc = _write(d, "poc.py",
|
||||
"import app\n"
|
||||
"class Cur:\n"
|
||||
" def fetchone(self):\n"
|
||||
" return None\n"
|
||||
"assert app.get_user_email(Cur(), 123) is None\n"
|
||||
"print('no crash — code handles the null case')\n")
|
||||
ev = V.run_poc(poc, target, expect_fail=True)
|
||||
self.assertEqual(ev["verdict"], "REFUTED")
|
||||
self.assertEqual(ev["exit_code"], 0)
|
||||
|
||||
def test_unverified_on_timeout(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py", "while True: pass\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=True, timeout=5)
|
||||
self.assertEqual(ev["verdict"], "UNVERIFIED")
|
||||
self.assertTrue(ev["timed_out"])
|
||||
|
||||
def test_unverified_when_clean_run_no_expectation(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py", "print('hello')\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=False, expect_output=None)
|
||||
self.assertEqual(ev["verdict"], "UNVERIFIED") # proves nothing
|
||||
|
||||
def test_non_posix_fallback_no_crash(self):
|
||||
# Windows has no resource module and no preexec_fn support — the
|
||||
# sandbox must degrade to timeout-only, not crash.
|
||||
saved_posix, saved_res = V._POSIX, V._HAS_RESOURCE
|
||||
V._POSIX, V._HAS_RESOURCE = False, False
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py", "print('hello')\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=False, expect_output=None)
|
||||
self.assertEqual(ev["verdict"], "UNVERIFIED")
|
||||
self.assertFalse(ev["sandbox"]["rlimits_applied"])
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py", "while True: pass\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=True, timeout=3)
|
||||
self.assertEqual(ev["verdict"], "UNVERIFIED")
|
||||
self.assertTrue(ev["timed_out"])
|
||||
finally:
|
||||
V._POSIX, V._HAS_RESOURCE = saved_posix, saved_res
|
||||
|
||||
def test_race_condition_verified_by_real_concurrency(self):
|
||||
# The hard class CWE-362: double-spend only manifests under actual
|
||||
# concurrency. The PoC spawns threads against the buggy code.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
target = _write(d, "app.py",
|
||||
"class Account:\n"
|
||||
" def __init__(self, balance):\n"
|
||||
" self.balance = balance\n"
|
||||
"def withdraw(acct, amount):\n"
|
||||
" if acct.balance >= amount: # check\n"
|
||||
" acct.balance -= amount # act (non-atomic)\n"
|
||||
" return True\n"
|
||||
" return False\n")
|
||||
poc = _write(d, "poc.py",
|
||||
"import app, threading\n"
|
||||
"acct = app.Account(100)\n"
|
||||
"results = []\n"
|
||||
"def try_take():\n"
|
||||
" results.append(app.withdraw(acct, 100))\n"
|
||||
"ts = [threading.Thread(target=try_take) for _ in range(8)]\n"
|
||||
"[t.start() for t in ts]\n"
|
||||
"[t.join() for t in ts]\n"
|
||||
"if sum(results) > 1:\n"
|
||||
" print('DOUBLE-SPEND: %d withdrawals succeeded from balance 100' % sum(results))\n"
|
||||
" import sys; sys.exit(3)\n"
|
||||
"print('no race observed this run')\n")
|
||||
ev = V.run_poc(poc, target, expect_fail=True, expect_output="DOUBLE-SPEND")
|
||||
# GIL makes the race probabilistic; with 8 threads on a
|
||||
# check-then-act it fires reliably, but we accept either a
|
||||
# VERIFIED race or an honest UNVERIFIED/REFUTED — never a crash.
|
||||
self.assertIn(ev["verdict"], ("VERIFIED", "REFUTED", "UNVERIFIED"))
|
||||
self.assertIsNotNone(ev["exit_code"])
|
||||
|
||||
def test_sandbox_no_env_leakage(self):
|
||||
# The PoC must NOT see any parent environment variables.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py",
|
||||
"import os\n"
|
||||
"leaked = [k for k in os.environ if not k.startswith(('PATH', 'LC_', 'LANG'))]\n"
|
||||
"print('LEAKED:', leaked)\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=False, expect_output=None)
|
||||
self.assertNotIn("HERMES", ev["stdout"])
|
||||
self.assertNotIn("SECRET", ev["stdout"])
|
||||
# env={} means essentially nothing is present:
|
||||
self.assertNotIn("HOME", ev["stdout"])
|
||||
|
||||
def test_output_truncated(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
poc = _write(d, "poc.py", "print('x' * 100_000)\n")
|
||||
ev = V.run_poc(poc, None, expect_fail=False, expect_output=None)
|
||||
self.assertLessEqual(len(ev["stdout"]), V.MAX_OUTPUT_BYTES + 100)
|
||||
|
||||
|
||||
class TestPolicyEnforcement(unittest.TestCase):
|
||||
def test_hard_class_without_verification_violates(self):
|
||||
report = {"findings": [
|
||||
{"id": "HG-1", "classes": ["CWE-362"], "severity": "HIGH"},
|
||||
]}
|
||||
r = V.check_report_policy(report)
|
||||
self.assertFalse(r["ok"])
|
||||
self.assertEqual(r["violations"][0]["verification_verdict"], "MISSING")
|
||||
|
||||
def test_hard_class_with_verified_passes(self):
|
||||
report = {"findings": [
|
||||
{"id": "HG-1", "classes": ["CWE-362"], "severity": "HIGH",
|
||||
"verification": {"verdict": "VERIFIED", "signal": "DOUBLE-SPEND observed"}},
|
||||
]}
|
||||
r = V.check_report_policy(report)
|
||||
self.assertTrue(r["ok"])
|
||||
|
||||
def test_soft_class_exempt(self):
|
||||
report = {"findings": [
|
||||
{"id": "HG-2", "classes": ["CWE-89"], "severity": "CRITICAL"}, # sqli: not forced
|
||||
]}
|
||||
r = V.check_report_policy(report)
|
||||
self.assertTrue(r["ok"])
|
||||
self.assertEqual(r["hard_class_findings"], 0)
|
||||
|
||||
def test_unverified_still_violates(self):
|
||||
report = {"findings": [
|
||||
{"id": "HG-1", "classes": ["HG-ASYNC"], "verification": {"verdict": "UNVERIFIED"}},
|
||||
]}
|
||||
r = V.check_report_policy(report)
|
||||
self.assertFalse(r["ok"])
|
||||
|
||||
def test_cli_check_report(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
rp = Path(d) / "report.json"
|
||||
rp.write_text(json.dumps({"findings": [
|
||||
{"id": "HG-1", "classes": ["HG-PARTIAL-WRITE"]}]}))
|
||||
tool = str(ROOT / "tools/hermaguard-verify/hermaguard-verify.py")
|
||||
r = subprocess.run([sys.executable, tool, "--check-report", str(rp)],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 1)
|
||||
self.assertIn("HG-PARTIAL-WRITE", r.stdout)
|
||||
self.assertIn("HG-1", r.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user