feat(loop): accept only measured clean candidate commits
This commit is contained in:
1 parent
49752daa7c
commit
3f70b8dd48
3 files changed
+1217
No files matched your search
@@ -16,7 +16,11 @@ jobs:
|
|||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: "3.12"
|
python-version: "3.12"
|
||||||
|
- name: Isolamento Linux para testes do loop
|
||||||
|
run: sudo apt-get update && sudo apt-get install -y bubblewrap
|
||||||
- name: Lint das skills, agentes e hooks
|
- name: Lint das skills, agentes e hooks
|
||||||
run: python3 qa/lint_skills.py
|
run: python3 qa/lint_skills.py
|
||||||
- name: Testes do gate e dos hooks
|
- name: Testes do gate e dos hooks
|
||||||
run: python3 qa/test_gate.py
|
run: python3 qa/test_gate.py
|
||||||
|
- name: Testes comportamentais de catálogo, roteador, adapters e loop
|
||||||
|
run: python3 -m unittest discover -s qa -p 'test_*.py' -q
|
||||||
Executable
+388
@@ -0,0 +1,388 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""End-to-end controller tests on disposable repositories, using the real gate/evaluator."""
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
SCRIPTS = Path(__file__).resolve().parent.parent / "scripts"
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
import goal_loop as loop
|
||||||
|
|
||||||
|
|
||||||
|
def git(repo, *args):
|
||||||
|
p = subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True)
|
||||||
|
if p.returncode:
|
||||||
|
raise AssertionError(p.stderr)
|
||||||
|
return p.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
class GoalLoopTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
root = Path(self.tmp.name)
|
||||||
|
self.repo = root / "app"
|
||||||
|
self.repo.mkdir()
|
||||||
|
self.state = root / "state"
|
||||||
|
self.bin = root / "bin"
|
||||||
|
self.bin.mkdir()
|
||||||
|
scanner = self.bin / "gitleaks"
|
||||||
|
scanner.write_text("#!/bin/sh\nexit 0\n")
|
||||||
|
scanner.chmod(0o755)
|
||||||
|
self.env = patch.dict(os.environ, {"DW_STATE_HOME": str(self.state),
|
||||||
|
"PATH": str(self.bin) + os.pathsep + os.environ["PATH"]})
|
||||||
|
self.env.start()
|
||||||
|
self.addCleanup(self.env.stop)
|
||||||
|
git(self.repo, "init", "-q")
|
||||||
|
git(self.repo, "config", "user.name", "test")
|
||||||
|
git(self.repo, "config", "user.email", "[email protected]")
|
||||||
|
(self.repo / "calc.py").write_text("VALUE = 0\n")
|
||||||
|
(self.repo / "verify.py").write_text("import runpy\nassert runpy.run_path('calc.py')['VALUE'] >= 0\n")
|
||||||
|
self.evaluator = root / "evaluator.py"
|
||||||
|
self.evaluator.write_text(
|
||||||
|
"import json, os, pathlib, runpy\n"
|
||||||
|
"value = runpy.run_path('calc.py')['VALUE']\n"
|
||||||
|
"folder = pathlib.Path(os.environ['DW_REPORT_DIR'])\n"
|
||||||
|
"evidence = folder / 'result.json'\n"
|
||||||
|
"evidence.write_text(json.dumps({'measured': value}))\n"
|
||||||
|
"print(json.dumps({'schema_version': 1, 'metric': 'quality', 'value': value, "
|
||||||
|
"'unit': 'points', 'direction': 'maximize', 'checks': {'regression': 'pass'}, "
|
||||||
|
"'evidence': [str(evidence)]}))\n")
|
||||||
|
self.config = self.repo / ".dev-workflows.toml"
|
||||||
|
self.write_config()
|
||||||
|
git(self.repo, "add", "-A")
|
||||||
|
git(self.repo, "commit", "-qm", "baseline")
|
||||||
|
self.original = git(self.repo, "rev-parse", "HEAD")
|
||||||
|
|
||||||
|
def write_config(self, target=2, iterations=4, stagnant=2, tier=0, billing="claude_subscription"):
|
||||||
|
checksum = hashlib.sha256(self.evaluator.read_bytes()).hexdigest()
|
||||||
|
self.config.write_text(
|
||||||
|
f"[commands]\ntest = \"python3 verify.py\"\nlint = \"true\"\ntypecheck = \"true\"\n"
|
||||||
|
f"[loop]\nenabled = true\nharness = \"claude\"\ntier = {tier}\n"
|
||||||
|
f"max_iterations = {iterations}\nmax_minutes = 5\nmax_stagnant = {stagnant}\n"
|
||||||
|
"min_improvement = 1\nmax_paid_usd = 0\n"
|
||||||
|
f"[loop.goal]\nname = \"quality\"\nunit = \"points\"\ndirection = \"maximize\"\ntarget = {target}\n"
|
||||||
|
"[loop.evaluator]\n"
|
||||||
|
f"argv = [\"python3\", \"{self.evaluator}\"]\n"
|
||||||
|
f"trusted_sha256 = \"{checksum}\"\nrequired_checks = [\"test\", \"lint\"]\n"
|
||||||
|
"[models.roles.implementer]\n"
|
||||||
|
f"model_key = \"sonnet\"\nbilling_mode = \"{billing}\"\nmanual_unranked_approval = true\n")
|
||||||
|
if tier >= 2:
|
||||||
|
with self.config.open("a") as config:
|
||||||
|
for role in ("planner", "reviewer"):
|
||||||
|
config.write(f"[models.roles.{role}]\nmodel_key = \"opus\"\n"
|
||||||
|
"billing_mode = \"claude_subscription\"\nmanual_unranked_approval = true\n")
|
||||||
|
if tier >= 3:
|
||||||
|
config.write("[models.roles.security-auditor]\nmodel_key = \"opus\"\n"
|
||||||
|
"billing_mode = \"claude_subscription\"\nmanual_unranked_approval = true\n")
|
||||||
|
|
||||||
|
def runner(self, behavior):
|
||||||
|
def invoke(tool, role, prompt, cwd, model_key, timeout=900, env=None):
|
||||||
|
if role in ("planner", "reviewer", "security-auditor"):
|
||||||
|
return {"returncode": 0, "output": '{"verdict":"pass","findings":[]}', "usage": {}}
|
||||||
|
value = int((Path(cwd) / "calc.py").read_text().split("=")[1].strip())
|
||||||
|
next_value = behavior(value, cwd)
|
||||||
|
(Path(cwd) / "calc.py").write_text(f"VALUE = {next_value}\n")
|
||||||
|
return {"returncode": 0, "output": "changed one behavior", "usage": {}}
|
||||||
|
return patch.dict(sys.modules, {"harness_runner": types.SimpleNamespace(invoke=invoke)})
|
||||||
|
|
||||||
|
def test_plan_does_not_create_worktree_or_invoke_agent(self):
|
||||||
|
before = git(self.repo, "worktree", "list", "--porcelain")
|
||||||
|
result = loop.plan(self.config)
|
||||||
|
self.assertEqual(result["baseline"], "not_run")
|
||||||
|
self.assertEqual(result["gate_preflight"]["test"], "run")
|
||||||
|
self.assertEqual(git(self.repo, "worktree", "list", "--porcelain"), before)
|
||||||
|
self.assertFalse(self.state.exists())
|
||||||
|
|
||||||
|
def test_two_measured_changes_reach_target_without_touching_user_branch(self):
|
||||||
|
with self.runner(lambda value, _: value + 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "success", result)
|
||||||
|
self.assertEqual([a["result"] for a in result["attempts"]], ["accepted", "accepted"])
|
||||||
|
self.assertEqual(result["best_score"], "2")
|
||||||
|
self.assertEqual(git(self.repo, "rev-parse", "HEAD"), self.original)
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
self.assertEqual((Path(result["best_worktree"]) / "calc.py").read_text(), "VALUE = 2\n")
|
||||||
|
self.assertNotIn(".dev-workflows/gate.json", git(Path(result["best_worktree"]), "ls-files"))
|
||||||
|
self.assertEqual(result["attempts"][1]["gate"]["checks"]["test"], "pass")
|
||||||
|
|
||||||
|
def test_limit_preserves_best_improvement_without_claiming_target(self):
|
||||||
|
self.write_config(target=3, iterations=2)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "target-three")
|
||||||
|
with self.runner(lambda value, _: value + 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["best_score"], "2")
|
||||||
|
self.assertEqual([a["result"] for a in result["attempts"]], ["accepted", "accepted"])
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
|
||||||
|
def test_measured_improvement_can_remove_obsolete_code(self):
|
||||||
|
(self.repo / "legacy.py").write_text("OLD = True\n")
|
||||||
|
self.evaluator.write_text(self.evaluator.read_text().replace(
|
||||||
|
"value = runpy.run_path('calc.py')['VALUE']",
|
||||||
|
"value = runpy.run_path('calc.py')['VALUE'] + int(not pathlib.Path('legacy.py').exists())"))
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
git(self.repo, "add", "-A")
|
||||||
|
git(self.repo, "commit", "-qm", "legacy-baseline")
|
||||||
|
def invoke(tool, role, prompt, cwd, model_key, timeout=900, env=None):
|
||||||
|
(Path(cwd) / "legacy.py").unlink()
|
||||||
|
return {"returncode": 0, "output": "removed legacy", "usage": {}}
|
||||||
|
with patch.dict(sys.modules, {"harness_runner": types.SimpleNamespace(invoke=invoke)}):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "success", result)
|
||||||
|
self.assertFalse((Path(result["best_worktree"]) / "legacy.py").exists())
|
||||||
|
self.assertTrue((self.repo / "legacy.py").exists())
|
||||||
|
|
||||||
|
|
||||||
|
def test_baseline_at_target_exits_without_editing(self):
|
||||||
|
self.write_config(target=0)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "zero-target")
|
||||||
|
with self.runner(lambda _value, _cwd: self.fail("agente não deveria ser chamado")):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "success", result)
|
||||||
|
self.assertEqual(result["iteration"], 0)
|
||||||
|
self.assertEqual(result["attempts"], [])
|
||||||
|
|
||||||
|
def test_evaluator_cannot_modify_original_checkout(self):
|
||||||
|
self.evaluator.write_text(
|
||||||
|
self.evaluator.read_text() +
|
||||||
|
f"open({str(self.repo / 'calc.py')!r}, 'w').write('VALUE = 999\\n')\n")
|
||||||
|
self.write_config(target=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "protected-evaluator")
|
||||||
|
with self.runner(lambda value, _: value + 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "error", result)
|
||||||
|
self.assertIn("avaliador", result["error"])
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
|
||||||
|
|
||||||
|
def test_gate_failure_rejects_regression_without_replacing_best(self):
|
||||||
|
calls = 0
|
||||||
|
def changes(value, _):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
return -1 if calls == 1 else 1
|
||||||
|
self.write_config(target=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "target-one")
|
||||||
|
with self.runner(changes):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "success", result)
|
||||||
|
self.assertEqual([a["result"] for a in result["attempts"]], ["rejected", "accepted"])
|
||||||
|
self.assertIn("quality gate", result["attempts"][0]["reason"])
|
||||||
|
self.assertEqual(result["best_score"], "1")
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
|
||||||
|
def test_incomplete_gate_never_counts_as_success(self):
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "one-iteration")
|
||||||
|
# No gitleaks in PATH: a security check is unverified, not green.
|
||||||
|
with patch.dict(os.environ, {"PATH": os.environ["PATH"].replace(str(self.bin) + os.pathsep, "")}), \
|
||||||
|
self.runner(lambda value, _: 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["best_score"], "0")
|
||||||
|
self.assertIn("quality gate", result["attempts"][0]["reason"])
|
||||||
|
|
||||||
|
def test_tampered_config_in_candidate_is_rejected(self):
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "one-iteration")
|
||||||
|
def tamper(value, cwd):
|
||||||
|
with (Path(cwd) / ".dev-workflows.toml").open("a") as f:
|
||||||
|
f.write("\n# altered by agent\n")
|
||||||
|
return 1
|
||||||
|
with self.runner(tamper):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["attempts"][0]["result"], "rejected")
|
||||||
|
self.assertIn("protegido", result["attempts"][0]["reason"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_redirected_git_pointer_cannot_stage_on_original_branch(self):
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "one-iteration")
|
||||||
|
original = git(self.repo, "rev-parse", "HEAD")
|
||||||
|
|
||||||
|
def redirect(_value, cwd):
|
||||||
|
(Path(cwd) / ".git").write_text(f"gitdir: {self.repo / '.git'}\n")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
with self.runner(redirect):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["attempts"][0]["result"], "rejected")
|
||||||
|
self.assertIn("ponteiro .git", result["attempts"][0]["reason"])
|
||||||
|
self.assertEqual(git(self.repo, "rev-parse", "HEAD"), original)
|
||||||
|
self.assertEqual(git(self.repo, "status", "--porcelain"), "")
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
|
||||||
|
def test_ignored_agent_file_cannot_inflate_accepted_score(self):
|
||||||
|
(self.repo / ".gitignore").write_text("score.txt\n")
|
||||||
|
self.evaluator.write_text(self.evaluator.read_text().replace(
|
||||||
|
"value = runpy.run_path('calc.py')['VALUE']",
|
||||||
|
"value = int(pathlib.Path('score.txt').read_text()) if pathlib.Path('score.txt').exists() else 0"))
|
||||||
|
self.write_config(target=2, iterations=1)
|
||||||
|
git(self.repo, "add", "-A")
|
||||||
|
git(self.repo, "commit", "-qm", "ignore-uncommitted-score")
|
||||||
|
|
||||||
|
def inflate(_value, cwd):
|
||||||
|
(Path(cwd) / "score.txt").write_text("2\n")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
with self.runner(inflate):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["best_score"], "0")
|
||||||
|
self.assertEqual(result["attempts"][0]["result"], "rejected")
|
||||||
|
self.assertIn("score 0", result["attempts"][0]["reason"])
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 0\n")
|
||||||
|
|
||||||
|
def test_missing_evidence_cannot_claim_improvement(self):
|
||||||
|
self.evaluator.write_text(self.evaluator.read_text().replace(
|
||||||
|
"'evidence': [str(evidence)]", "'evidence': [str(evidence)] if value == 0 else []"))
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "evidence-required")
|
||||||
|
with self.runner(lambda _value, _cwd: 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "paused", result)
|
||||||
|
self.assertEqual(result["best_score"], "0")
|
||||||
|
self.assertEqual(result["attempts"][0]["result"], "paused")
|
||||||
|
self.assertIn("evidência", result["attempts"][0]["reason"])
|
||||||
|
|
||||||
|
def test_dirty_tree_is_refused_and_preserved(self):
|
||||||
|
(self.repo / "calc.py").write_text("VALUE = 99\n")
|
||||||
|
with self.assertRaisesRegex(loop.LoopError, "alterações"):
|
||||||
|
loop.new_run(self.config)
|
||||||
|
self.assertFalse(self.state.exists())
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 99\n")
|
||||||
|
|
||||||
|
def test_status_and_stop_work_after_unrelated_user_edit(self):
|
||||||
|
self.write_config(iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "one-iteration")
|
||||||
|
with self.runner(lambda value, _: value + 1):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted")
|
||||||
|
(self.repo / "calc.py").write_text("VALUE = 99\n")
|
||||||
|
run_id = result["run_id"]
|
||||||
|
for action in ("status", "stop"):
|
||||||
|
with patch("sys.stdout", new_callable=io.StringIO) as captured:
|
||||||
|
exit_code = loop.main([action, "--config", str(self.config), "--run", run_id])
|
||||||
|
self.assertEqual(exit_code, 0)
|
||||||
|
self.assertEqual(json.loads(captured.getvalue())["best_score"], "1")
|
||||||
|
self.assertTrue((loop.state_root(self.repo) / run_id / "stop.requested").is_file())
|
||||||
|
self.assertEqual((self.repo / "calc.py").read_text(), "VALUE = 99\n")
|
||||||
|
|
||||||
|
def test_independent_review_failure_rejects_measured_gain(self):
|
||||||
|
self.write_config(target=1, tier=2, iterations=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "review-required")
|
||||||
|
def invoke(tool, role, prompt, cwd, model_key, timeout=900, env=None):
|
||||||
|
if role == "implementer":
|
||||||
|
(Path(cwd) / "calc.py").write_text("VALUE = 1\n")
|
||||||
|
return {"returncode": 0, "output": "changed behavior", "usage": {}}
|
||||||
|
return {"returncode": 0, "output": '{"verdict":"fail","findings":["regression"]}'
|
||||||
|
if role == "reviewer" else "plan", "usage": {}}
|
||||||
|
actual_gate = loop.gate_result
|
||||||
|
# Keep a real passing gate; tier-2 scanners are not installed in the disposable app.
|
||||||
|
def gate_with_available_checks(repo, base, tier, required, changes, deadline=None):
|
||||||
|
return actual_gate(repo, base, 0, required, changes, deadline)
|
||||||
|
with (patch.dict(sys.modules, {"harness_runner": types.SimpleNamespace(invoke=invoke)}),
|
||||||
|
patch.object(loop, "gate_result", side_effect=gate_with_available_checks)):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "exhausted", result)
|
||||||
|
self.assertEqual(result["best_score"], "0")
|
||||||
|
self.assertEqual(result["attempts"][0]["gate"]["checks"]["test"], "pass")
|
||||||
|
self.assertIn("reviewer", result["attempts"][0]["reason"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_resume_after_interrupted_attempt_does_not_repeat_it(self):
|
||||||
|
self.write_config(target=1)
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "target-one")
|
||||||
|
def interrupt(_, __):
|
||||||
|
raise KeyboardInterrupt
|
||||||
|
with self.runner(interrupt), self.assertRaises(KeyboardInterrupt):
|
||||||
|
loop.new_run(self.config)
|
||||||
|
home = loop.state_root(self.repo)
|
||||||
|
folder = next(p for p in home.iterdir() if p.is_dir())
|
||||||
|
with self.runner(lambda value, _: 1):
|
||||||
|
result = loop.resume_run(self.config, folder.name)
|
||||||
|
self.assertEqual(result["status"], "success", result)
|
||||||
|
self.assertEqual([a["result"] for a in result["attempts"]], ["interrupted", "accepted"])
|
||||||
|
self.assertEqual(result["attempts"][-1]["iteration"], 2)
|
||||||
|
|
||||||
|
def test_fixed_zen_model_cannot_bypass_account_catalog(self):
|
||||||
|
self.write_config(target=1, billing="zen")
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "zen-config")
|
||||||
|
with self.runner(lambda value, _: 1), self.assertRaisesRegex(loop.LoopError, "OpenCode ou OMP"):
|
||||||
|
loop.new_run(self.config)
|
||||||
|
|
||||||
|
def test_unknown_paid_usage_blocks_resume_without_reauthorizing(self):
|
||||||
|
self.write_config(target=1, iterations=1)
|
||||||
|
self.config.write_text(self.config.read_text().replace("max_paid_usd = 0", "max_paid_usd = 1"))
|
||||||
|
git(self.repo, "add", ".dev-workflows.toml")
|
||||||
|
git(self.repo, "commit", "-qm", "paid-budget")
|
||||||
|
selected = {"model_key": "opencode/test", "billing_mode": "zen",
|
||||||
|
"balance_confirmed": True, "estimated_native_usd": "0.1"}
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def invoke(_tool, _role, _prompt, cwd, _model_key, timeout=900, env=None):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
(Path(cwd) / "calc.py").write_text("VALUE = 1\n")
|
||||||
|
return {"returncode": 0, "output": "changed behavior", "usage": None}
|
||||||
|
|
||||||
|
with (patch.object(loop, "role_model", return_value=selected),
|
||||||
|
patch.object(loop, "authorize"),
|
||||||
|
patch.dict(sys.modules, {"harness_runner": types.SimpleNamespace(invoke=invoke)})):
|
||||||
|
result = loop.new_run(self.config)
|
||||||
|
self.assertEqual(result["status"], "paused", result)
|
||||||
|
self.assertEqual(result["spent_paid_usd"], "0")
|
||||||
|
state = json.loads((loop.state_root(self.repo) / result["run_id"] / "state.json").read_text())
|
||||||
|
self.assertTrue(state["unreconciled_paid_call"])
|
||||||
|
with self.assertRaisesRegex(loop.LoopError, "concilie"):
|
||||||
|
loop.resume_run(self.config, result["run_id"])
|
||||||
|
self.assertEqual(calls, 1)
|
||||||
|
|
||||||
|
def test_go_requires_all_three_windows_and_disables_balance_fallback(self):
|
||||||
|
config = {"loop": {"max_paid_usd": 0}}
|
||||||
|
row = {"billing_mode": "go", "quota_window_remaining": {"5h": "1", "weekly": "1", "monthly": "1"},
|
||||||
|
"quota_window_debit": {"5h": "0.2", "weekly": "0.2", "monthly": "0.2"},
|
||||||
|
"use_balance_fallback": True}
|
||||||
|
with self.assertRaisesRegex(loop.SafetyPause, "fallback"):
|
||||||
|
loop.authorize(config, row, loop.Decimal(0))
|
||||||
|
row["use_balance_fallback"] = False
|
||||||
|
row["quota_window_remaining"]["weekly"] = "0"
|
||||||
|
with self.assertRaisesRegex(loop.SafetyPause, "três janelas"):
|
||||||
|
loop.authorize(config, row, loop.Decimal(0))
|
||||||
|
row["quota_window_remaining"]["weekly"] = "1"
|
||||||
|
loop.authorize(config, row, loop.Decimal(0))
|
||||||
|
|
||||||
|
def test_zen_needs_live_interactive_consent_not_stdin_yes(self):
|
||||||
|
config = {"loop": {"max_paid_usd": 1}}
|
||||||
|
row = {"billing_mode": "zen", "model_key": "opencode/test",
|
||||||
|
"estimated_native_usd": "0.2", "balance_confirmed": True}
|
||||||
|
with patch("sys.stdin", io.StringIO("SIM\n")), self.assertRaisesRegex(loop.SafetyPause, "interativa"):
|
||||||
|
loop.authorize(config, row, loop.Decimal(0))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
Executable
+825
@@ -0,0 +1,825 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Opt-in, evaluator-driven experiments in detached git worktrees.
|
||||||
|
|
||||||
|
The user's branch is never reset, staged, merged, or published. Agent output is
|
||||||
|
untrusted; only the external evaluator and current quality-gate report decide.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
from decimal import Decimal, InvalidOperation
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import tomllib
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
GATE = ROOT / "scripts" / "quality-gate"
|
||||||
|
MODEL_CATALOG = ROOT / "scripts" / "model_catalog.py"
|
||||||
|
|
||||||
|
|
||||||
|
class LoopError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class SafetyPause(LoopError):
|
||||||
|
"""The next model invocation is not authorized or cannot be budgeted."""
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def command(argv, cwd, timeout=60, env=None):
|
||||||
|
try:
|
||||||
|
return subprocess.run(argv, cwd=cwd, env=env, text=True, capture_output=True, timeout=timeout)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise LoopError(f"falha ao executar {argv[0]}: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def sandbox_binary():
|
||||||
|
bwrap = next((path for path in ("/usr/bin/bwrap", "/bin/bwrap") if Path(path).is_file()), None)
|
||||||
|
if not bwrap:
|
||||||
|
raise LoopError("bubblewrap indisponível: não executar código do projeto sem isolamento")
|
||||||
|
return bwrap
|
||||||
|
|
||||||
|
|
||||||
|
def preflight_isolation(project):
|
||||||
|
"""Prove namespace support without modifying the project or invoking a model."""
|
||||||
|
result = command([sandbox_binary(), "--die-with-parent", "--new-session", "--unshare-net",
|
||||||
|
"--unshare-pid", "--ro-bind", "/", "/", "--dev-bind", "/dev", "/dev",
|
||||||
|
"--proc", "/proc", "--", "/usr/bin/true"], project, timeout=15)
|
||||||
|
if result.returncode:
|
||||||
|
raise LoopError(f"isolamento bubblewrap indisponível: {result.stderr.strip()[:300]}")
|
||||||
|
|
||||||
|
|
||||||
|
def isolated_command(argv, cwd, timeout=60, env=None, writable=()):
|
||||||
|
"""Run untrusted project checks with no network or write access outside mounts."""
|
||||||
|
bwrap = sandbox_binary()
|
||||||
|
repo = Path(cwd).resolve(strict=True)
|
||||||
|
home = repo / ".dev-workflows" / "sandbox-home"
|
||||||
|
home.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
child_env = dict(os.environ if env is None else env)
|
||||||
|
child_env.update(HOME=str(home), XDG_CACHE_HOME=str(home), TMPDIR=str(home),
|
||||||
|
PYTHONDONTWRITEBYTECODE="1")
|
||||||
|
marker = repo / ".git"
|
||||||
|
if marker.is_symlink() or not marker.is_file():
|
||||||
|
raise LoopError("sandbox exige ponteiro .git de worktree regular")
|
||||||
|
args = [bwrap, "--die-with-parent", "--new-session", "--unshare-net", "--unshare-pid",
|
||||||
|
"--ro-bind", "/", "/", "--dev-bind", "/dev", "/dev", "--proc", "/proc",
|
||||||
|
"--bind", str(repo), str(repo), "--ro-bind", str(marker), str(marker)]
|
||||||
|
for directory in writable:
|
||||||
|
resolved = Path(directory).resolve(strict=True)
|
||||||
|
if not resolved.is_dir():
|
||||||
|
raise LoopError(f"diretório de relatório inválido: {directory}")
|
||||||
|
args.extend(("--bind", str(resolved), str(resolved)))
|
||||||
|
args.extend(("--", *argv))
|
||||||
|
return command(args, repo, timeout, child_env)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def git(root, *args):
|
||||||
|
result = command(["git", *args], root)
|
||||||
|
if result.returncode:
|
||||||
|
raise LoopError(f"git {args[0]} falhou: {result.stderr.strip()[:300]}")
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def digest(path):
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as source:
|
||||||
|
for block in iter(lambda: source.read(1024 * 1024), b""):
|
||||||
|
h.update(block)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def number(value, name):
|
||||||
|
if isinstance(value, bool):
|
||||||
|
raise LoopError(f"{name}: número inválido")
|
||||||
|
try:
|
||||||
|
d = Decimal(str(value))
|
||||||
|
except (InvalidOperation, ValueError) as exc:
|
||||||
|
raise LoopError(f"{name}: número inválido") from exc
|
||||||
|
if not d.is_finite():
|
||||||
|
raise LoopError(f"{name}: número não finito")
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def bounded_timeout(limit, deadline=None):
|
||||||
|
if deadline is None:
|
||||||
|
return limit
|
||||||
|
remaining = deadline - time.time()
|
||||||
|
if remaining <= 0:
|
||||||
|
raise LoopError("limite global de tempo esgotado")
|
||||||
|
return min(limit, remaining)
|
||||||
|
|
||||||
|
|
||||||
|
def within(path, directory):
|
||||||
|
return path == directory or directory in path.parents
|
||||||
|
|
||||||
|
|
||||||
|
def config_at(config_path):
|
||||||
|
path = Path(config_path).resolve(strict=True)
|
||||||
|
with path.open("rb") as source:
|
||||||
|
config = tomllib.load(source)
|
||||||
|
top = Path(git(path.parent, "rev-parse", "--show-toplevel")).resolve()
|
||||||
|
if not within(path, top):
|
||||||
|
raise LoopError("configuração deve estar no repositório avaliado")
|
||||||
|
loop = config.get("loop", {})
|
||||||
|
goal = loop.get("goal", {})
|
||||||
|
evaluator = loop.get("evaluator", {})
|
||||||
|
if not goal.get("name") or not goal.get("unit") or goal.get("direction") not in ("maximize", "minimize"):
|
||||||
|
raise LoopError("[loop.goal] requer name, unit e direction maximize|minimize")
|
||||||
|
target = number(goal.get("target"), "target")
|
||||||
|
argv = evaluator.get("argv")
|
||||||
|
if not isinstance(argv, list) or not argv or any(not isinstance(x, str) or not x for x in argv):
|
||||||
|
raise LoopError("[loop.evaluator].argv deve ser uma lista de argumentos")
|
||||||
|
# An interpreter may precede the evaluator. The executable script must not
|
||||||
|
# come from the editable checkout; a bare command in PATH is not auditable.
|
||||||
|
candidate = argv[1] if Path(argv[0]).name in ("python", "python3", "node", "bun") and len(argv) > 1 else argv[0]
|
||||||
|
script = Path(candidate)
|
||||||
|
if not script.is_absolute() or not script.is_file():
|
||||||
|
raise LoopError("avaliador precisa ser arquivo absoluto existente fora do worktree")
|
||||||
|
script = script.resolve(strict=True)
|
||||||
|
if within(script, top):
|
||||||
|
raise LoopError("avaliador não pode residir no checkout editável")
|
||||||
|
expected = evaluator.get("trusted_sha256", "")
|
||||||
|
if not re.fullmatch(r"[0-9a-fA-F]{64}", expected) or digest(script) != expected.lower():
|
||||||
|
raise LoopError("hash aprovado do avaliador não corresponde ao arquivo")
|
||||||
|
required = evaluator.get("required_checks", [])
|
||||||
|
if not isinstance(required, list) or not required or any(not isinstance(x, str) or not x for x in required):
|
||||||
|
raise LoopError("required_checks deve listar os checks obrigatórios do gate")
|
||||||
|
if evaluator.get("require_gate_verdict", "pass") != "pass":
|
||||||
|
raise LoopError("loop exige veredito pass no quality gate")
|
||||||
|
tier = loop.get("tier", 2)
|
||||||
|
if type(tier) is not int or tier not in range(4):
|
||||||
|
raise LoopError("[loop].tier deve estar entre 0 e 3")
|
||||||
|
loop["tier"] = tier
|
||||||
|
for key in ("max_iterations", "max_minutes", "max_stagnant"):
|
||||||
|
if type(loop.get(key)) is not int or loop[key] < 1:
|
||||||
|
raise LoopError(f"[loop].{key} deve ser inteiro positivo")
|
||||||
|
if number(loop.get("min_improvement"), "min_improvement") <= 0:
|
||||||
|
raise LoopError("min_improvement deve ser positivo")
|
||||||
|
if number(loop.get("max_paid_usd", 0), "max_paid_usd") < 0:
|
||||||
|
raise LoopError("max_paid_usd não pode ser negativo")
|
||||||
|
if loop.get("harness") not in ("claude", "opencode", "omp"):
|
||||||
|
raise LoopError("[loop].harness deve ser claude, opencode ou omp")
|
||||||
|
if not git(top, "status", "--porcelain", "--untracked-files=all") == "":
|
||||||
|
raise LoopError("working tree do usuário tem alterações; salve/snapshot antes de iniciar")
|
||||||
|
return path, top, config, script, target
|
||||||
|
|
||||||
|
|
||||||
|
def state_root(project):
|
||||||
|
default = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / "dev-workflows"
|
||||||
|
root = Path(os.environ.get("DW_STATE_HOME", default)).expanduser().resolve()
|
||||||
|
if within(root, project) or within(project, root):
|
||||||
|
raise LoopError("estado do controlador deve ficar fora do checkout do projeto")
|
||||||
|
return root / hashlib.sha256(os.fsencode(project)).hexdigest()[:20]
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_home(root):
|
||||||
|
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
os.chmod(root, 0o700)
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def locked(home):
|
||||||
|
prepare_home(home)
|
||||||
|
with (home / "lock").open("a+") as f:
|
||||||
|
try:
|
||||||
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError as exc:
|
||||||
|
raise LoopError("já existe uma execução ativa neste projeto") from exc
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(f, fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
def save(folder, state):
|
||||||
|
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
path = folder / "state.json"
|
||||||
|
tmp = folder / f".{uuid.uuid4().hex}.json"
|
||||||
|
with tmp.open("x") as out:
|
||||||
|
os.chmod(tmp, 0o600)
|
||||||
|
json.dump(state, out, ensure_ascii=False, indent=2)
|
||||||
|
out.flush()
|
||||||
|
os.fsync(out.fileno())
|
||||||
|
os.replace(tmp, path)
|
||||||
|
|
||||||
|
|
||||||
|
def load(home, run_id):
|
||||||
|
if not re.fullmatch(r"[0-9a-f]{32}", run_id):
|
||||||
|
raise LoopError("ID de execução inválido")
|
||||||
|
path = home / run_id / "state.json"
|
||||||
|
if not path.is_file():
|
||||||
|
raise LoopError("execução não encontrada")
|
||||||
|
with path.open() as f:
|
||||||
|
state = json.load(f)
|
||||||
|
return path.parent, state
|
||||||
|
|
||||||
|
|
||||||
|
def evaluator_result(cfg, script, repo, folder, deadline=None):
|
||||||
|
if digest(script) != cfg["loop"]["evaluator"]["trusted_sha256"].lower():
|
||||||
|
raise LoopError("avaliador protegido foi alterado")
|
||||||
|
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["DW_REPORT_DIR"] = str(folder)
|
||||||
|
argv = cfg["loop"]["evaluator"]["argv"]
|
||||||
|
result = isolated_command(argv, repo, bounded_timeout(
|
||||||
|
int(cfg["loop"]["evaluator"].get("timeout_seconds", 600)), deadline), env, writable=[folder])
|
||||||
|
if result.returncode:
|
||||||
|
raise LoopError(f"avaliador retornou {result.returncode}: {result.stderr.strip()[:300]}")
|
||||||
|
if digest(script) != cfg["loop"]["evaluator"]["trusted_sha256"].lower():
|
||||||
|
raise LoopError("avaliador protegido foi alterado durante a avaliação")
|
||||||
|
try:
|
||||||
|
data = json.loads(result.stdout, parse_constant=lambda _: (_ for _ in ()).throw(ValueError("NaN")))
|
||||||
|
score = number(data["value"], "score")
|
||||||
|
except (ValueError, KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||||
|
raise LoopError("avaliador não produziu JSON com score válido") from exc
|
||||||
|
goal = cfg["loop"]["goal"]
|
||||||
|
if (data.get("schema_version") != 1 or data.get("metric") != goal["name"]
|
||||||
|
or data.get("unit") != goal["unit"] or data.get("direction") != goal["direction"]):
|
||||||
|
raise LoopError("métrica/unidade/direção divergente da meta")
|
||||||
|
checks = data.get("checks")
|
||||||
|
if not isinstance(checks, dict) or not checks or any(x != "pass" for x in checks.values()):
|
||||||
|
raise LoopError("avaliador não aprovou todos os seus checks")
|
||||||
|
paths = data.get("evidence")
|
||||||
|
if not isinstance(paths, list) or not paths:
|
||||||
|
raise LoopError("avaliador não forneceu evidência verificável")
|
||||||
|
evidence = []
|
||||||
|
for raw in paths:
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise LoopError("caminho inválido de evidência")
|
||||||
|
p = Path(raw).resolve(strict=False)
|
||||||
|
if not p.is_file() or not within(p, folder.resolve()):
|
||||||
|
raise LoopError("evidência deve existir fora do worktree no diretório de relatórios")
|
||||||
|
evidence.append({"file": str(p), "sha256": digest(p)})
|
||||||
|
return score, evidence, checks
|
||||||
|
|
||||||
|
|
||||||
|
def attained(score, target, direction):
|
||||||
|
return score >= target if direction == "maximize" else score <= target
|
||||||
|
|
||||||
|
|
||||||
|
def improved(score, best, minimum, direction):
|
||||||
|
return score - best >= minimum if direction == "maximize" else best - score >= minimum
|
||||||
|
|
||||||
|
|
||||||
|
def worktree(project, folder, ref):
|
||||||
|
if folder.exists():
|
||||||
|
raise LoopError(f"worktree já existe: {folder}")
|
||||||
|
folder.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
git(project, "-c", "core.hooksPath=/dev/null", "worktree", "add", "--detach", str(folder), ref)
|
||||||
|
return folder
|
||||||
|
|
||||||
|
def assert_worktree(project, repo, expected_head, marker_sha256=None):
|
||||||
|
"""Reject redirected Git pointers or a branch before controller Git writes."""
|
||||||
|
marker = repo / ".git"
|
||||||
|
if marker.is_symlink() or not marker.is_file():
|
||||||
|
raise LoopError("ponteiro .git do worktree foi removido ou substituído")
|
||||||
|
try:
|
||||||
|
text = marker.read_text(encoding="utf-8")
|
||||||
|
except (OSError, UnicodeError) as exc:
|
||||||
|
raise LoopError("ponteiro .git do worktree ilegível") from exc
|
||||||
|
match = re.fullmatch(r"gitdir: ([^\r\n]+)\r?\n?", text)
|
||||||
|
if not match:
|
||||||
|
raise LoopError("ponteiro .git do worktree inválido")
|
||||||
|
marker_hash = digest(marker)
|
||||||
|
if marker_sha256 is not None and marker_hash != marker_sha256:
|
||||||
|
raise LoopError("ponteiro .git do worktree mudou durante a tentativa")
|
||||||
|
admin = Path(match.group(1))
|
||||||
|
admin = (repo / admin).resolve() if not admin.is_absolute() else admin.resolve()
|
||||||
|
common = Path(git(project, "rev-parse", "--git-common-dir"))
|
||||||
|
common = (project / common).resolve() if not common.is_absolute() else common.resolve()
|
||||||
|
if not admin.is_dir() or admin.parent != common / "worktrees":
|
||||||
|
raise LoopError("gitdir do worktree não pertence ao repositório original")
|
||||||
|
actual = command(["git", "rev-parse", "--absolute-git-dir"], repo)
|
||||||
|
if actual.returncode or Path(actual.stdout.strip()).resolve() != admin:
|
||||||
|
raise LoopError("ponteiro .git redireciona o worktree")
|
||||||
|
detached = command(["git", "symbolic-ref", "-q", "HEAD"], repo)
|
||||||
|
if detached.returncode != 1 or git(repo, "rev-parse", "HEAD") != expected_head:
|
||||||
|
raise LoopError("worktree não está no commit detached esperado")
|
||||||
|
return marker_hash
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def changed_paths(repo, base):
|
||||||
|
changed = set(git(repo, "diff", "--name-only", base).splitlines())
|
||||||
|
changed.update(git(repo, "ls-files", "--others", "--exclude-standard").splitlines())
|
||||||
|
return sorted(p for p in changed if p and not p.startswith(".dev-workflows/"))
|
||||||
|
|
||||||
|
|
||||||
|
def fingerprint(repo, names):
|
||||||
|
h = hashlib.sha256()
|
||||||
|
for name in names:
|
||||||
|
p = repo / name
|
||||||
|
h.update(os.fsencode(name))
|
||||||
|
h.update(b"\0")
|
||||||
|
if p.is_symlink():
|
||||||
|
h.update(os.fsencode(os.readlink(p)))
|
||||||
|
elif p.is_file():
|
||||||
|
h.update(digest(p).encode())
|
||||||
|
h.update(str(p.stat().st_mode & 0o777).encode())
|
||||||
|
else:
|
||||||
|
h.update(b"deleted")
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def gate_result(repo, base, tier, required, changes, deadline=None):
|
||||||
|
t = time.time_ns()
|
||||||
|
result = isolated_command([sys.executable, str(GATE), "--dir", str(repo), "--base", base,
|
||||||
|
"--tier", str(tier)], repo, bounded_timeout(3600, deadline))
|
||||||
|
report_path = repo / ".dev-workflows" / "gate.json"
|
||||||
|
if not report_path.is_file() or report_path.stat().st_mtime_ns < t:
|
||||||
|
raise LoopError("quality gate não gerou relatório atual")
|
||||||
|
report = json.loads(report_path.read_text())
|
||||||
|
if result.returncode or report.get("verdict") != "pass" or report.get("base") != base or report.get("tier") != tier:
|
||||||
|
raise LoopError(f"quality gate não aprovou o diff: {report.get('verdict', 'sem relatório')}")
|
||||||
|
checks = {c.get("name"): c.get("status") for c in report.get("checks", [])}
|
||||||
|
if any(checks.get(name) != "pass" for name in required):
|
||||||
|
raise LoopError("checks obrigatórios ausentes, não verificados ou falhos")
|
||||||
|
deleted = set(git(repo, "diff", "--name-only", "--diff-filter=D", base).splitlines())
|
||||||
|
if not set(changes).difference(deleted).issubset(set(report.get("changed_files", []))):
|
||||||
|
raise LoopError("quality gate não cobriu todos os arquivos alterados")
|
||||||
|
return {"verdict": report["verdict"], "tier": tier, "checks": checks,
|
||||||
|
"base": base, "changed_files": report["changed_files"], "sha256": digest(report_path)}
|
||||||
|
|
||||||
|
|
||||||
|
def role_model(config, tool, role, cwd):
|
||||||
|
models = config.get("models", {})
|
||||||
|
role_cfg = models.get("roles", {}).get(role, {})
|
||||||
|
if role_cfg.get("model_key"):
|
||||||
|
if not role_cfg.get("manual_unranked_approval", False):
|
||||||
|
raise LoopError(f"{role}: modelo fixado sem aprovação explícita unranked")
|
||||||
|
if tool != "claude" or role_cfg.get("billing_mode") != "claude_subscription":
|
||||||
|
raise LoopError("modelo fixo não substitui descoberta de disponibilidade/conta em OpenCode ou OMP")
|
||||||
|
model_key = role_cfg["model_key"]
|
||||||
|
if model_key not in ("haiku", "sonnet", "opus", "inherit"):
|
||||||
|
raise LoopError("Claude por assinatura aceita apenas aliases de modelo suportados")
|
||||||
|
return {"model_key": model_key, "billing_mode": "claude_subscription",
|
||||||
|
"score": None, "reason": "unranked: fixado explicitamente pelo usuário"}
|
||||||
|
providers = models.get("allowed_providers")
|
||||||
|
if not isinstance(providers, list) or not providers or any(not isinstance(x, str) for x in providers):
|
||||||
|
raise LoopError("[models].allowed_providers deve listar os providers autorizados da conta")
|
||||||
|
argv = [sys.executable, str(MODEL_CATALOG), "route", "--tool", tool, "--role", role,
|
||||||
|
"--providers", ",".join(providers), "--format", "json",
|
||||||
|
"--budget-usd", str(number(config["loop"].get("max_paid_usd", 0), "max_paid_usd"))]
|
||||||
|
if models.get("benchmark_metric"):
|
||||||
|
argv.extend(["--benchmark-metric", models["benchmark_metric"]])
|
||||||
|
if models.get("bands"):
|
||||||
|
argv.extend(["--bands-json", json.dumps(models["bands"])])
|
||||||
|
for key, flag in (("max_price_age_hours", "--max-price-age-hours"),
|
||||||
|
("max_score_age_days", "--max-score-age-days")):
|
||||||
|
if key in models:
|
||||||
|
argv.extend([flag, str(models[key])])
|
||||||
|
if role_cfg.get("band"):
|
||||||
|
argv.extend(["--band", role_cfg["band"]])
|
||||||
|
if "max_loss_points" in role_cfg:
|
||||||
|
argv.extend(["--max-loss-points", str(role_cfg["max_loss_points"])])
|
||||||
|
if "tokens" in role_cfg:
|
||||||
|
argv.extend(["--tokens", json.dumps(role_cfg["tokens"])])
|
||||||
|
if (models.get("allow_proxy_score") and role_cfg.get("allow_proxy_score", True)
|
||||||
|
and config["loop"]["tier"] < 3):
|
||||||
|
argv.append("--allow-proxy-score")
|
||||||
|
if models.get("cache_dir"):
|
||||||
|
argv.extend(["--cache-dir", models["cache_dir"]])
|
||||||
|
p = command(argv, cwd)
|
||||||
|
if p.returncode:
|
||||||
|
raise LoopError(f"roteamento {role}: {p.stderr.strip()[:300]}")
|
||||||
|
try:
|
||||||
|
result = json.loads(p.stdout)
|
||||||
|
selected = result.get("selected")
|
||||||
|
if not selected or not selected.get("model_key"):
|
||||||
|
raise LoopError(f"{role}: sem modelo elegível: {result.get('reason', 'sem nota')}")
|
||||||
|
except (ValueError, AttributeError) as exc:
|
||||||
|
raise LoopError("roteador não devolveu JSON válido") from exc
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def authorize(config, model, remaining):
|
||||||
|
billing = model.get("billing_mode")
|
||||||
|
if billing in ("free", "subscription", "claude_subscription"):
|
||||||
|
return
|
||||||
|
if billing == "go":
|
||||||
|
quota = model.get("quota_window_remaining")
|
||||||
|
debit = model.get("quota_window_debit")
|
||||||
|
if (not isinstance(quota, dict) or not isinstance(debit, dict) or
|
||||||
|
any(k not in quota or k not in debit or number(quota[k], k) < number(debit[k], k)
|
||||||
|
for k in ("5h", "weekly", "monthly"))):
|
||||||
|
raise SafetyPause("Go: cota não confirmada nas três janelas; sem fallback para saldo Zen")
|
||||||
|
if model.get("use_balance_fallback") is not False:
|
||||||
|
raise SafetyPause("Go: fallback para saldo Zen desconhecido ou cobrado; não autorizado")
|
||||||
|
return
|
||||||
|
if billing not in ("zen", "payg"):
|
||||||
|
raise SafetyPause("modalidade de cobrança desconhecida; despacho não autorizado")
|
||||||
|
label = "Zen" if billing == "zen" else "API paga"
|
||||||
|
if model.get("balance_confirmed") is not True:
|
||||||
|
raise SafetyPause(f"{label}: saldo ou permissão de cobrança da conta não confirmado")
|
||||||
|
cap = number(config["loop"].get("max_paid_usd", 0), "max_paid_usd")
|
||||||
|
estimate = model.get("estimated_native_usd")
|
||||||
|
if cap == 0 or estimate is None or number(estimate, "estimated_native_usd") > cap - remaining:
|
||||||
|
raise SafetyPause(f"{label}: orçamento explícito e estimativa nativa atual são obrigatórios")
|
||||||
|
if not sys.stdin.isatty():
|
||||||
|
raise SafetyPause(f"{label}: autorização interativa exigida imediatamente antes do gasto")
|
||||||
|
question = (f"Autorizar {model['model_key']} via {label}, estimativa USD {estimate}, "
|
||||||
|
f"limite restante USD {cap - remaining}? Digite SIM: ")
|
||||||
|
if input(question) != "SIM":
|
||||||
|
raise SafetyPause(f"gasto {label} não autorizado")
|
||||||
|
|
||||||
|
def mark_call_pending(state, folder, model):
|
||||||
|
if model.get("billing_mode") in ("zen", "payg", "go"):
|
||||||
|
state["unreconciled_paid_call"] = True
|
||||||
|
save(folder, state)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def selected_roles(config, cwd):
|
||||||
|
tier = config["loop"]["tier"]
|
||||||
|
roles = ["implementer"]
|
||||||
|
if tier >= 2:
|
||||||
|
roles = ["planner", "implementer", "reviewer"]
|
||||||
|
if tier >= 3:
|
||||||
|
roles.append("security-auditor")
|
||||||
|
return {role: role_model(config, config["loop"]["harness"], role, cwd) for role in roles}
|
||||||
|
|
||||||
|
|
||||||
|
def same_route(previous, current):
|
||||||
|
return all(previous.get(key) == current.get(key) for key in
|
||||||
|
("model_key", "billing_mode", "provider", "variant", "estimated_native_usd", "native_tariff"))
|
||||||
|
|
||||||
|
|
||||||
|
def verified_model(config, state, role):
|
||||||
|
previous = state["route_plan"][role]
|
||||||
|
if previous.get("billing_mode") == "claude_subscription":
|
||||||
|
return previous # CLI authentication is checked immediately before dispatch.
|
||||||
|
try:
|
||||||
|
current = role_model(config, config["loop"]["harness"], role, Path(state["project"]))
|
||||||
|
except LoopError as exc:
|
||||||
|
raise SafetyPause(f"{role}: disponibilidade, preço ou quota não revalidada: {exc}") from exc
|
||||||
|
if not same_route(previous, current):
|
||||||
|
raise SafetyPause(f"{role}: rota ou tarifa mudou; novo plan e aprovação necessários")
|
||||||
|
if current.get("billing_mode") == "go":
|
||||||
|
for key in ("5h", "weekly", "monthly"):
|
||||||
|
current["quota_window_remaining"][key] = str(min(
|
||||||
|
number(previous["quota_window_remaining"][key], key),
|
||||||
|
number(current["quota_window_remaining"][key], key)))
|
||||||
|
state["route_plan"][role] = current
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
|
def report_run(state):
|
||||||
|
return {k: state.get(k) for k in
|
||||||
|
("run_id", "status", "baseline_score", "best_score", "target", "iteration",
|
||||||
|
"best_commit", "best_worktree", "spent_paid_usd", "elapsed_seconds",
|
||||||
|
"baseline_evidence", "baseline_checks", "baseline_gate", "attempts",
|
||||||
|
"route_plan", "gate_sha256", "evaluator_sha256", "error")}
|
||||||
|
|
||||||
|
|
||||||
|
def plan(config_path):
|
||||||
|
path, project, cfg, script, target = config_at(config_path)
|
||||||
|
preflight_isolation(project)
|
||||||
|
routes = {}
|
||||||
|
roles = (["planner", "implementer", "reviewer", "security-auditor"] if cfg["loop"]["tier"] >= 3
|
||||||
|
else ["planner", "implementer", "reviewer"] if cfg["loop"]["tier"] >= 2 else ["implementer"])
|
||||||
|
preflight = command([sys.executable, str(GATE), "--dir", str(project),
|
||||||
|
"--base", git(project, "rev-parse", "HEAD"),
|
||||||
|
"--tier", str(cfg["loop"]["tier"]), "--list"], project)
|
||||||
|
if preflight.returncode:
|
||||||
|
raise LoopError(f"quality gate indisponível: {preflight.stderr.strip()[:300]}")
|
||||||
|
planned = {fields[1]: " ".join(fields[3:]) for line in preflight.stdout.splitlines()
|
||||||
|
if len(fields := line.split()) >= 4 and fields[0].startswith("T")}
|
||||||
|
gate = {name: planned.get(name, "ausente") for name in cfg["loop"]["evaluator"]["required_checks"]}
|
||||||
|
for role in roles:
|
||||||
|
try:
|
||||||
|
routes[role] = role_model(cfg, cfg["loop"]["harness"], role, project)
|
||||||
|
except LoopError as exc:
|
||||||
|
routes[role] = {"status": "unavailable", "reason": str(exc)}
|
||||||
|
return {"config": str(path), "enabled": cfg["loop"].get("enabled", False), "root": str(project),
|
||||||
|
"evaluator_sha256": digest(script), "gate_sha256": digest(GATE),
|
||||||
|
"baseline": "not_run", "target": str(target),
|
||||||
|
"tier": cfg["loop"]["tier"], "routes": routes, "gate_preflight": gate,
|
||||||
|
"risks": ["modelos e permissões devem ser revalidados antes de cada chamada",
|
||||||
|
"worktree não substitui isolamento de processo e acesso a rede",
|
||||||
|
"preço OpenRouter não equivale à cobrança da ferramenta"],
|
||||||
|
"max_paid_usd": str(number(cfg["loop"].get("max_paid_usd", 0), "max_paid_usd"))}
|
||||||
|
|
||||||
|
|
||||||
|
def new_run(config_path):
|
||||||
|
path, project, cfg, script, target = config_at(config_path)
|
||||||
|
if cfg["loop"].get("enabled") is not True:
|
||||||
|
raise LoopError("loop desativado: habilite [loop].enabled explicitamente")
|
||||||
|
home = state_root(project)
|
||||||
|
with locked(home):
|
||||||
|
routes = selected_roles(cfg, project)
|
||||||
|
for model in routes.values():
|
||||||
|
if (model.get("billing_mode") in ("zen", "payg")
|
||||||
|
and number(cfg["loop"].get("max_paid_usd", 0), "max_paid_usd") == 0):
|
||||||
|
raise LoopError("modelo pago: gasto não autorizado; max_paid_usd=0")
|
||||||
|
base = git(project, "rev-parse", "HEAD")
|
||||||
|
run_id = uuid.uuid4().hex
|
||||||
|
folder = home / run_id
|
||||||
|
folder.mkdir(mode=0o700)
|
||||||
|
work = worktree(project, folder / "worktrees" / "baseline", base)
|
||||||
|
state = {"version": 1, "run_id": run_id, "project": str(project), "config": str(path),
|
||||||
|
"config_sha256": digest(path), "evaluator_sha256": digest(script),
|
||||||
|
"gate_sha256": digest(GATE), "base": base,
|
||||||
|
"best_commit": base, "best_worktree": str(work), "best_score": None,
|
||||||
|
"baseline_score": None,
|
||||||
|
"target": str(target), "iteration": 0, "spent_paid_usd": "0", "stagnant": 0,
|
||||||
|
"started_at": time.time(), "elapsed_seconds": 0, "status": "planned", "attempts": [],
|
||||||
|
"route_plan": routes}
|
||||||
|
save(folder, state)
|
||||||
|
try:
|
||||||
|
deadline = state["started_at"] + cfg["loop"]["max_minutes"] * 60
|
||||||
|
score, evidence, checks = evaluator_result(
|
||||||
|
cfg, script, work, folder / "evidence" / "baseline", deadline)
|
||||||
|
if changed_paths(work, base):
|
||||||
|
raise LoopError("avaliador modificou baseline; medição inicial inválida")
|
||||||
|
state.update(best_score=str(score), baseline_score=str(score), baseline_evidence=evidence,
|
||||||
|
baseline_checks=checks, status="baselined")
|
||||||
|
save(folder, state)
|
||||||
|
if attained(score, target, cfg["loop"]["goal"]["direction"]):
|
||||||
|
state["baseline_gate"] = gate_result(work, base, cfg["loop"]["tier"],
|
||||||
|
cfg["loop"]["evaluator"]["required_checks"], [], deadline)
|
||||||
|
state["status"] = "success"
|
||||||
|
save(folder, state)
|
||||||
|
else:
|
||||||
|
iterate(folder, state, cfg, script)
|
||||||
|
except (LoopError, OSError, ValueError) as exc:
|
||||||
|
state.update(status="error", error=str(exc))
|
||||||
|
save(folder, state)
|
||||||
|
return report_run(state)
|
||||||
|
|
||||||
|
|
||||||
|
def check_resumption(state, cfg, project, script, path):
|
||||||
|
if state["project"] != str(project) or state["config_sha256"] != digest(path):
|
||||||
|
raise LoopError("configuração/checkout mudou desde o início; não retomar automaticamente")
|
||||||
|
if (state["evaluator_sha256"] != digest(script) or state["gate_sha256"] != digest(GATE)
|
||||||
|
or git(project, "rev-parse", "HEAD") != state["base"]):
|
||||||
|
raise LoopError("avaliador, gate ou base Git mudaram; retomar exige nova avaliação")
|
||||||
|
if state["status"] not in ("exhausted", "paused", "running", "baselined"):
|
||||||
|
raise LoopError(f"execução com estado {state['status']} não pode ser retomada")
|
||||||
|
if state.get("unreconciled_paid_call"):
|
||||||
|
raise LoopError("tentativa paga com consumo/quota indeterminados; concilie antes de novo run")
|
||||||
|
routes = selected_roles(cfg, project)
|
||||||
|
if set(routes) != set(state["route_plan"]) or any(
|
||||||
|
not same_route(state["route_plan"][role], fresh) for role, fresh in routes.items()):
|
||||||
|
raise LoopError("catálogo, preço, conta ou modelos mudaram; não retomar sem novo plan")
|
||||||
|
if not Path(state["best_worktree"]).is_dir():
|
||||||
|
raise LoopError("worktree vencedor foi removido; não retomar")
|
||||||
|
|
||||||
|
|
||||||
|
def resume_run(config_path, run_id):
|
||||||
|
path, project, cfg, script, target = config_at(config_path)
|
||||||
|
home = state_root(project)
|
||||||
|
with locked(home):
|
||||||
|
folder, state = load(home, run_id)
|
||||||
|
check_resumption(state, cfg, project, script, path)
|
||||||
|
if (state["status"] == "running" and len(state["attempts"]) < state["iteration"]
|
||||||
|
and any(row.get("billing_mode") in ("zen", "payg", "go") for row in state["route_plan"].values())):
|
||||||
|
raise LoopError("tentativa paga interrompida: consumo/quota indeterminados; concilie antes de novo run")
|
||||||
|
if cfg["loop"].get("enabled") is not True:
|
||||||
|
raise LoopError("loop desativado")
|
||||||
|
if state["status"] == "running" and len(state["attempts"]) < state["iteration"]:
|
||||||
|
state["attempts"].append({"iteration": state["iteration"], "result": "interrupted", "reason": "tentativa interrompida, não reutilizada"})
|
||||||
|
state["status"] = "baselined"
|
||||||
|
save(folder, state)
|
||||||
|
try:
|
||||||
|
iterate(folder, state, cfg, script)
|
||||||
|
except (LoopError, OSError, ValueError) as exc:
|
||||||
|
state.update(status="error", error=str(exc))
|
||||||
|
save(folder, state)
|
||||||
|
return report_run(state)
|
||||||
|
|
||||||
|
|
||||||
|
def iterate(folder, state, cfg, script):
|
||||||
|
from harness_runner import invoke
|
||||||
|
|
||||||
|
minimum = number(cfg["loop"]["min_improvement"], "min_improvement")
|
||||||
|
target = number(state["target"], "target")
|
||||||
|
project = Path(state["project"])
|
||||||
|
path = Path(state["config"])
|
||||||
|
direction = cfg["loop"]["goal"]["direction"]
|
||||||
|
deadline = state["started_at"] + cfg["loop"]["max_minutes"] * 60
|
||||||
|
while state["iteration"] < cfg["loop"]["max_iterations"]:
|
||||||
|
if (folder / "stop.requested").exists():
|
||||||
|
state["status"] = "cancelled"
|
||||||
|
break
|
||||||
|
if time.time() >= deadline:
|
||||||
|
state["status"] = "exhausted"
|
||||||
|
break
|
||||||
|
if state["stagnant"] >= cfg["loop"]["max_stagnant"]:
|
||||||
|
state["status"] = "exhausted"
|
||||||
|
break
|
||||||
|
if (digest(path) != state["config_sha256"] or digest(script) != state["evaluator_sha256"]
|
||||||
|
or digest(GATE) != state["gate_sha256"]):
|
||||||
|
state["status"] = "paused"
|
||||||
|
state["error"] = "configuração, avaliador ou gate alterado durante a execução"
|
||||||
|
break
|
||||||
|
if git(project, "rev-parse", "HEAD") != state["base"] or git(project, "status", "--porcelain", "--untracked-files=all"):
|
||||||
|
state["status"] = "paused"
|
||||||
|
state["error"] = "checkout original alterado durante a execução; preservar mudanças do usuário"
|
||||||
|
break
|
||||||
|
step = state["iteration"] + 1
|
||||||
|
base = state["best_commit"]
|
||||||
|
work = worktree(project, folder / "worktrees" / f"iteration-{step:03}", base)
|
||||||
|
marker_hash = assert_worktree(project, work, base)
|
||||||
|
state.update(status="running", iteration=step)
|
||||||
|
save(folder, state)
|
||||||
|
attempt = {"iteration": step, "worktree": str(work), "from": base}
|
||||||
|
paid = number(state["spent_paid_usd"], "spent_paid_usd")
|
||||||
|
try:
|
||||||
|
tool = cfg["loop"]["harness"]
|
||||||
|
routes = state["route_plan"]
|
||||||
|
timeout = int(cfg["loop"].get("agent_timeout_seconds", 900))
|
||||||
|
prompt = (f"Use dev-router com tier mínimo {cfg['loop']['tier']}. Meta do aplicativo: "
|
||||||
|
f"{cfg['loop']['goal']['name']} {direction} {target} {cfg['loop']['goal']['unit']}. "
|
||||||
|
f"Score atual {state['best_score']}. Faça UMA hipótese pequena por iteração; "
|
||||||
|
"preserve testes e quality gate. Não modifique o avaliador, config, catálogos ou testes protegidos. "
|
||||||
|
"Não publique, instale dependências ou produza efeitos externos sem autorização. "
|
||||||
|
f"Feedback anterior: {state['attempts'][-1]['reason'] if state['attempts'] else 'nenhum'}. ")
|
||||||
|
if "planner" in routes:
|
||||||
|
model = verified_model(cfg, state, "planner")
|
||||||
|
authorize(cfg, model, paid)
|
||||||
|
mark_call_pending(state, folder, model)
|
||||||
|
planned = invoke(tool, "planner", prompt + "Planeje uma única hipótese; só leitura.", str(work),
|
||||||
|
model["model_key"], timeout=bounded_timeout(timeout, deadline),
|
||||||
|
env={"DEV_WORKFLOWS_AUTONOMOUS": "1"})
|
||||||
|
assert_worktree(project, work, base, marker_hash)
|
||||||
|
paid = charged(planned, model, cfg, state, folder)
|
||||||
|
if planned["returncode"]:
|
||||||
|
raise SafetyPause("planejamento falhou; nenhuma outra chamada autorizada")
|
||||||
|
prompt += "Plano (não é autorização): " + planned["output"][:3000] + ". "
|
||||||
|
model = verified_model(cfg, state, "implementer")
|
||||||
|
authorize(cfg, model, paid)
|
||||||
|
mark_call_pending(state, folder, model)
|
||||||
|
done = invoke(tool, "implementer", prompt + "Implemente e relate nível declarado e evidências.", str(work),
|
||||||
|
model["model_key"], timeout=bounded_timeout(timeout, deadline),
|
||||||
|
env={"DEV_WORKFLOWS_AUTONOMOUS": "1"})
|
||||||
|
paid = charged(done, model, cfg, state, folder)
|
||||||
|
if done["returncode"]:
|
||||||
|
raise SafetyPause("agente de implementação falhou; nenhuma outra chamada autorizada")
|
||||||
|
assert_worktree(project, work, base, marker_hash)
|
||||||
|
changes = changed_paths(work, base)
|
||||||
|
if not changes:
|
||||||
|
raise LoopError("agente não modificou código verificável")
|
||||||
|
if path.relative_to(project).as_posix() in changes or any(c.startswith(".dev-workflows/") for c in changes):
|
||||||
|
raise LoopError("tentativa alterou configuração ou estado protegido")
|
||||||
|
if (git(project, "rev-parse", "HEAD") != state["base"]
|
||||||
|
or git(project, "status", "--porcelain", "--untracked-files=all")):
|
||||||
|
raise LoopError("checkout original alterado durante a tentativa; preservar mudanças do usuário")
|
||||||
|
assert_worktree(project, work, base, marker_hash)
|
||||||
|
# This commit is only a candidate in a detached worktree, never a promotion.
|
||||||
|
git(work, "add", "-A", "--", ".", ":(exclude).dev-workflows/**")
|
||||||
|
git(work, "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false",
|
||||||
|
"-c", "user.name=Dev Workflows", "-c", "user.email=dev-workflows@localhost",
|
||||||
|
"commit", "-m", f"dev-workflows: candidate iteration {step}")
|
||||||
|
candidate_commit = git(work, "rev-parse", "HEAD")
|
||||||
|
assert_worktree(project, work, candidate_commit, marker_hash)
|
||||||
|
if git(project, "rev-parse", "HEAD") != state["base"]:
|
||||||
|
raise LoopError("checkout original alterado durante o snapshot")
|
||||||
|
|
||||||
|
# Gate/review the exact commit, not agent-created ignored files.
|
||||||
|
verified = worktree(project, folder / "verification" / "worktrees" / f"iteration-{step:03}", candidate_commit)
|
||||||
|
verified_marker = assert_worktree(project, verified, candidate_commit)
|
||||||
|
if changed_paths(verified, base) != changes:
|
||||||
|
raise LoopError("snapshot da tentativa diverge dos arquivos alterados")
|
||||||
|
before = fingerprint(verified, changes)
|
||||||
|
gate = gate_result(verified, base, cfg["loop"]["tier"],
|
||||||
|
cfg["loop"]["evaluator"]["required_checks"], changes, deadline)
|
||||||
|
assert_worktree(project, verified, candidate_commit, verified_marker)
|
||||||
|
if changed_paths(verified, base) != changes or before != fingerprint(verified, changes):
|
||||||
|
raise LoopError("código mudou durante o quality gate")
|
||||||
|
attempt["gate"] = gate
|
||||||
|
for role in ("reviewer", "security-auditor"):
|
||||||
|
if role in routes:
|
||||||
|
model = verified_model(cfg, state, role)
|
||||||
|
authorize(cfg, model, paid)
|
||||||
|
mark_call_pending(state, folder, model)
|
||||||
|
review = invoke(tool, role, "Revise exclusivamente o diff da tentativa. Responda JSON: "
|
||||||
|
'{"verdict":"pass|fail","findings":[]}.', str(verified), model["model_key"],
|
||||||
|
timeout=bounded_timeout(timeout, deadline),
|
||||||
|
env={"DEV_WORKFLOWS_AUTONOMOUS": "1"})
|
||||||
|
paid = charged(review, model, cfg, state, folder)
|
||||||
|
if review["returncode"]:
|
||||||
|
raise SafetyPause(f"{role} falhou; nenhuma outra chamada autorizada")
|
||||||
|
if review_verdict(review["output"]) != "pass":
|
||||||
|
raise LoopError(f"{role} rejeitou ou não concluiu a revisão")
|
||||||
|
assert_worktree(project, verified, candidate_commit, verified_marker)
|
||||||
|
if changed_paths(verified, base) != changes or fingerprint(verified, changes) != before:
|
||||||
|
raise LoopError("código mudou depois do gate/revisão")
|
||||||
|
|
||||||
|
# Score a second clean checkout: gate artifacts cannot bias the metric.
|
||||||
|
scored = worktree(project, folder / "scoring" / "worktrees" / f"iteration-{step:03}", candidate_commit)
|
||||||
|
scored_marker = assert_worktree(project, scored, candidate_commit)
|
||||||
|
try:
|
||||||
|
score, evidence, checks = evaluator_result(
|
||||||
|
cfg, script, scored, folder / "evidence" / f"iteration-{step:03}", deadline)
|
||||||
|
except LoopError as exc:
|
||||||
|
raise SafetyPause(f"avaliador inválido: {exc}") from exc
|
||||||
|
assert_worktree(project, scored, candidate_commit, scored_marker)
|
||||||
|
if (changed_paths(scored, candidate_commit) or digest(path) != state["config_sha256"]
|
||||||
|
or git(project, "rev-parse", "HEAD") != state["base"]):
|
||||||
|
raise LoopError("código/configuração/base mudou durante a avaliação")
|
||||||
|
attempt.update(score=str(score), evidence=evidence, evaluator_checks=checks)
|
||||||
|
best = number(state["best_score"], "best_score")
|
||||||
|
if not improved(score, best, minimum, direction):
|
||||||
|
raise LoopError(f"sem melhoria mínima sobre {best}: score {score}")
|
||||||
|
bounded_timeout(1, deadline)
|
||||||
|
state.update(best_commit=candidate_commit, best_worktree=str(scored), best_score=str(score), stagnant=0)
|
||||||
|
attempt.update(result="accepted", reason="melhoria medida em snapshot limpo e gate aprovado",
|
||||||
|
commit=candidate_commit)
|
||||||
|
if attained(score, target, direction):
|
||||||
|
state["status"] = "success"
|
||||||
|
except SafetyPause as exc:
|
||||||
|
attempt.update(result="paused", reason=str(exc))
|
||||||
|
state["status"] = "paused"
|
||||||
|
except RuntimeError as exc:
|
||||||
|
attempt.update(result="paused", reason=f"runtime recusou ou falhou: {exc}")
|
||||||
|
state["status"] = "paused"
|
||||||
|
except (LoopError, OSError, ValueError) as exc:
|
||||||
|
attempt.update(result="rejected", reason=str(exc))
|
||||||
|
state["stagnant"] += 1
|
||||||
|
state["attempts"].append(attempt)
|
||||||
|
state["elapsed_seconds"] = round(time.time() - state["started_at"], 2)
|
||||||
|
save(folder, state)
|
||||||
|
if state["status"] in ("success", "paused"):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
state["status"] = "exhausted"
|
||||||
|
if state["status"] == "running":
|
||||||
|
state["status"] = "exhausted"
|
||||||
|
save(folder, state)
|
||||||
|
|
||||||
|
|
||||||
|
def usage_cost(result, model):
|
||||||
|
value = (result.get("usage") or {}).get("cost_usd")
|
||||||
|
if model.get("billing_mode") in ("zen", "payg") and value is None:
|
||||||
|
raise SafetyPause("modelo pago: custo real da chamada não foi informado; pausa obrigatória")
|
||||||
|
cost = number(value, "cost_usd") if value is not None else Decimal(0)
|
||||||
|
if cost < 0:
|
||||||
|
raise SafetyPause("custo negativo informado pelo provedor; pausa obrigatória")
|
||||||
|
if model.get("billing_mode") == "go":
|
||||||
|
quota = model["quota_window_remaining"]
|
||||||
|
debit = model["quota_window_debit"]
|
||||||
|
for window in ("5h", "weekly", "monthly"):
|
||||||
|
quota[window] = str(number(quota[window], window) - number(debit[window], window))
|
||||||
|
return cost if model.get("billing_mode") in ("zen", "payg") else Decimal(0)
|
||||||
|
|
||||||
|
|
||||||
|
def charged(result, model, config, state, folder):
|
||||||
|
total = number(state["spent_paid_usd"], "spent_paid_usd") + usage_cost(result, model)
|
||||||
|
state["spent_paid_usd"] = str(total)
|
||||||
|
if model.get("billing_mode") in ("zen", "payg", "go"):
|
||||||
|
state["unreconciled_paid_call"] = False
|
||||||
|
save(folder, state)
|
||||||
|
if (model.get("billing_mode") in ("zen", "payg")
|
||||||
|
and total > number(config["loop"]["max_paid_usd"], "max_paid_usd")):
|
||||||
|
raise SafetyPause(f"modelo pago: custo real USD {total} excedeu limite aprovado; nenhuma outra chamada autorizada")
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def review_verdict(output):
|
||||||
|
try:
|
||||||
|
report = json.loads(output)
|
||||||
|
except ValueError:
|
||||||
|
return "invalid"
|
||||||
|
return report.get("verdict") if report.get("verdict") in ("pass", "fail") and isinstance(report.get("findings"), list) else "invalid"
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
parser.add_argument("action", choices=("plan", "run", "status", "resume", "stop"))
|
||||||
|
parser.add_argument("--config", default=".dev-workflows.toml")
|
||||||
|
parser.add_argument("--run", dest="run_id")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
try:
|
||||||
|
if args.action == "plan":
|
||||||
|
result = plan(args.config)
|
||||||
|
elif args.action == "run":
|
||||||
|
result = new_run(args.config)
|
||||||
|
else:
|
||||||
|
if not args.run_id:
|
||||||
|
raise LoopError("--run <id> obrigatório")
|
||||||
|
path = Path(args.config).resolve(strict=True)
|
||||||
|
project = Path(git(path.parent, "rev-parse", "--show-toplevel")).resolve()
|
||||||
|
home = state_root(project)
|
||||||
|
if args.action == "resume":
|
||||||
|
result = resume_run(args.config, args.run_id)
|
||||||
|
else:
|
||||||
|
folder, state = load(home, args.run_id)
|
||||||
|
if args.action == "stop":
|
||||||
|
(folder / "stop.requested").touch(exist_ok=True)
|
||||||
|
result = report_run(state)
|
||||||
|
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||||
|
return 0 if args.action in ("plan", "status", "stop") or result.get("status") == "success" else 1
|
||||||
|
except (LoopError, FileNotFoundError, tomllib.TOMLDecodeError) as exc:
|
||||||
|
print(json.dumps({"status": "error", "reason": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in new issue
Block a user