feat(catalog): audit model availability and route by verified costs

This commit is contained in:
Evandro committed 2026-09-25 14:21:41 -03:00
1 parent f77b0b440d
commit 2662a80bd8
6 files changed
+26218

No files matched your search

+374
View File
@@ -0,0 +1,374 @@
#!/usr/bin/env python3
"""Behavior tests for model catalog imports and reports (stdlib only)."""
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from decimal import Decimal
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "scripts"))
import model_catalog as catalog
class CatalogImportTests(unittest.TestCase):
def test_openrouter_keeps_per_token_units_and_missing_cache_is_not_free(self):
payload = {"data": [{"id": "z-ai/glm-5.3", "name": "GLM", "pricing": {
"prompt": "0.0000014", "completion": "0.0000044", "input_cache_read": "0.00000026"
}}]}
prices = catalog.parse_openrouter(payload)["z-ai/glm-5.3"]
self.assertEqual(prices["price_unit"], "USD/token")
self.assertEqual(prices["price_per_token_usd"]["input"], "0.0000014")
self.assertEqual(Decimal(prices["display_usd_per_1m"]["input"]), Decimal("1.4"))
self.assertEqual(Decimal(prices["display_usd_per_1m"]["cache_read"]), Decimal("0.26"))
self.assertIsNone(prices["display_usd_per_1m"]["cache_write"])
self.assertIsNone(catalog.reference_cost(prices["price_per_token_usd"], {"cache_write": 1}))
self.assertEqual(catalog.reference_cost(prices["price_per_token_usd"], {"input": 1_000_000, "output": 10_000}), Decimal("1.444"))
def test_negative_public_tariff_keeps_valid_models_and_is_not_costed(self):
payload = {"data": [
{"id": "openrouter/auto-beta", "pricing": {"prompt": "-0.000001", "completion": "0.1"}},
{"id": "openai/valid", "pricing": {"prompt": "0.000002", "completion": "0.000003"}},
]}
parsed = catalog.parse_openrouter(payload)
self.assertEqual(catalog.reference_cost(parsed["openai/valid"]["price_per_token_usd"],
{"input": 1_000_000}), Decimal("2"))
invalid = parsed["openrouter/auto-beta"]
self.assertEqual(invalid["conditions"]["unsupported_negative_prices"]["input"], "-0.000001")
self.assertIsNone(invalid["price_per_token_usd"]["input"])
self.assertIsNone(catalog.reference_cost(invalid["price_per_token_usd"], {"input": 1}))
with self.assertRaises(ValueError):
catalog.parse_openrouter(payload, {"openai/valid": {"price_per_token_usd": {"input": "-1"},
"source_url": "https://example.test/pricing", "captured_at": "2026-09-25T00:00:00Z"}})
def test_openrouter_price_overrides_preserve_conditions_and_unit(self):
payload = {"data": [{"id": "openai/gpt-6-luna", "pricing": {"prompt": "0.1", "completion": "0.5"}, "context_length": 272000}]}
overrides = {"openai/gpt-6-luna": {
"display_usd_per_1m": {"input": "2.25", "output": "7.5"},
"context_length": 272000, "ttl": "1h", "endpoint": "https://example.test/v2",
"source_url": "https://example.test/pricing", "captured_at": "2026-09-25T00:00:00Z"
}}
record = catalog.parse_openrouter(payload, overrides)["openai/gpt-6-luna"]
self.assertEqual(record["price_per_token_usd"]["input"], "0.00000225")
self.assertEqual(Decimal(record["display_usd_per_1m"]["output"]), Decimal("7.5"))
self.assertEqual(record["conditions"]["context_length"], 272000)
self.assertEqual(record["conditions"]["ttl"], "1h")
self.assertEqual(record["conditions"]["endpoint"], "https://example.test/v2")
with self.assertRaises(ValueError):
catalog.parse_openrouter(payload, {"missing/model": {"display_usd_per_1m": {"input": "1"}}})
def test_context_and_cache_ttl_overrides_choose_the_matching_price_tier(self):
payload = {"data": [{"id": "openai/gpt-6-luna", "pricing": {
"prompt": "0.0000001", "completion": "0.0000005",
"input_cache_write": "0.0000009", "input_cache_write_1h": "0.0000005"
}}]}
tier = {"min_context_tokens": 272001, "price_per_token_usd": {"input": "0.0000002", "output": "0.0000008"},
"source_url": "https://openrouter.ai/docs/pricing", "captured_at": "2026-09-25T00:00:00Z"}
record = catalog.parse_openrouter(payload, {"openai/gpt-6-luna": {"context_tiers": [tier]}})["openai/gpt-6-luna"]
below = catalog.openrouter_price_for_profile(record, {"context_length": 272000})
above = catalog.openrouter_price_for_profile(record, {"context_length": 272001})
self.assertEqual(below["price_per_token_usd"]["input"], "0.0000001")
self.assertEqual(above["price_per_token_usd"]["input"], "0.0000002")
self.assertEqual(above["applied_context_tier"], 272001)
self.assertEqual(catalog.reference_cost(record["price_per_token_usd"], {"cache_write": 100, "cache_write_ttl": "1h"}), Decimal("0.00005"))
def test_provider_catalog_preserves_exact_variants_without_guessing(self):
rows = catalog.parse_provider_catalog({"data": [
{"id": "model-x", "name": "X", "variants": ["high", "max"]},
{"id": "model-y"}
]}, "opencode", catalog.ZEN_URL)
self.assertEqual([r["model_key"] for r in rows], ["opencode/model-x#high", "opencode/model-x#max", "opencode/model-y"])
self.assertTrue(all(r["availability"]["authenticated"] is None for r in rows))
with self.assertRaises(ValueError):
catalog.parse_provider_catalog({"data": [{"id": "bad/id"}]}, "opencode", catalog.ZEN_URL)
def test_aa_import_requires_consent_provenance_and_one_consistent_v15_record(self):
snapshot = {
"metric_id": catalog.AA_METRIC, "version": "1.5", "ingestion": "manual_audited",
"terms_accepted": True, "source_url": catalog.AA_URL, "attribution": "Artificial Analysis Coding Agent Index",
"captured_at": "2026-09-25T00:00:00Z", "records": [{
"provider": "opencode", "model_id": "gpt-6-luna", "variant": "max", "score": "41",
"agent": "Codex", "harness": "Codex", "settings": {"effort": "max"},
"match_grade": "proxy", "evidence_url": catalog.AA_URL
}]
}
row = catalog.validate_aa_snapshot(snapshot)[0]
self.assertEqual((row["metric_id"], row["version"], row["score"], row["match_grade"]), (catalog.AA_METRIC, "1.5", "41", "proxy"))
wrong_index = dict(snapshot, metric_id="artificial_analysis_intelligence_index")
with self.assertRaises(ValueError):
catalog.validate_aa_snapshot(wrong_index)
duplicate = dict(snapshot, records=snapshot["records"] * 2)
with self.assertRaisesRegex(ValueError, "inconsistente"):
catalog.validate_aa_snapshot(duplicate)
no_consent = dict(snapshot, terms_accepted=False)
with self.assertRaises(ValueError):
catalog.validate_aa_snapshot(no_consent)
def test_local_discovery_never_returns_diagnostics_or_credential_text(self):
class Result:
returncode = 0
stdout = "opencode/model-a#high\nAPI_KEY=do-not-store\n"
stderr = "secret=do-not-print"
result = catalog.discover_local_models("opencode", runner=lambda *args, **kwargs: Result())
self.assertEqual(result["models"], ["model-a#high"])
self.assertNotIn("do-not-store", json.dumps(result))
self.assertNotIn("do-not-print", json.dumps(result))
def test_omp_json_is_local_listing_only_and_never_proves_entitlement(self):
payload = {"models": [{"provider": "openai", "id": "gpt-6-luna", "thinking": ["high"]}]}
rows = catalog.parse_omp_models(payload, "openai")
self.assertEqual(rows[0]["model_key"], "openai/gpt-6-luna")
self.assertIsNone(rows[0]["availability"]["authenticated"])
self.assertIsNone(rows[0]["availability"]["entitled"])
class Result:
returncode = 0
stdout = json.dumps(payload)
stderr = "account-private-value"
calls = []
result = catalog.discover_omp_models("openai", runner=lambda args, **kwargs: (calls.append(args) or Result()))
self.assertEqual(calls, [["omp", "models", "openai", "--json"]])
self.assertEqual(result["models"], ["openai/gpt-6-luna", "openai/gpt-6-luna#high"])
self.assertNotIn("account-private-value", json.dumps(result))
def test_omp_keeps_slash_ids_and_only_lists_advertised_thinking(self):
payload = {"models": [
{"provider": "openai", "id": "anthropic/claude-x",
"selector": "openai/anthropic/claude-x", "thinking": ["low", "high"]}
]}
rows = catalog.parse_omp_models(payload, "openai")
self.assertEqual([row["model_key"] for row in rows], ["openai/anthropic/claude-x"])
self.assertEqual(rows[0]["model_id"], "anthropic/claude-x")
class Result:
returncode = 0
stdout = json.dumps(payload)
stderr = ""
discovered = catalog.discover_omp_models("openai", runner=lambda *args, **kwargs: Result())
self.assertEqual(discovered["models"], [
"openai/anthropic/claude-x", "openai/anthropic/claude-x#high", "openai/anthropic/claude-x#low"
])
with self.assertRaises(ValueError):
catalog.parse_omp_models({"models": [{"provider": "openai", "id": "anthropic/claude-x",
"selector": "openai/claude-x"}]}, "openai")
def test_openrouter_public_long_context_override_affects_reference_cost(self):
record = catalog.parse_openrouter({"data": [{"id": "anthropic/model-x", "pricing": {
"prompt": "0.000001", "completion": "0.000002",
"overrides": [{"min_prompt_tokens": 272000, "prompt": "0.000003", "completion": "0.000004"}]
}}]})["anthropic/model-x"]
selected = catalog.openrouter_price_for_profile(record, {"input": 272000})
self.assertEqual(selected["price_per_token_usd"]["input"], "0.000003")
self.assertEqual(catalog.reference_cost(selected["price_per_token_usd"],
{"input": 272000, "output": 10}), Decimal("0.81604"))
def test_omp_sync_requires_manual_account_and_balance_evidence(self):
with tempfile.TemporaryDirectory() as cache:
availability_path = os.path.join(cache, "availability.json")
native_path = os.path.join(cache, "overrides.json")
aa_path = os.path.join(cache, "aa.json")
timestamp = catalog.utc_now()
with open(availability_path, "w", encoding="utf-8") as stream:
json.dump({
"tool": "omp", "ingestion": "manual_audited", "user_confirmed": True,
"captured_at": timestamp, "records": [{
"provider": "openai", "model_id": "gpt-6-luna", "variant": "high",
"local_listed": True, "authenticated": True, "entitled": True,
"verified_at": timestamp, "source": "manual audited OMP account",
"balance_confirmed": True, "balance_verified_at": timestamp,
}],
}, stream)
with open(native_path, "w", encoding="utf-8") as stream:
json.dump({"native_tariffs": {"openai/gpt-6-luna#high": {
"billing_mode": "omp_payg", "account_confirmed": True,
"source_url": "https://provider.example/pricing", "captured_at": timestamp,
"price_per_token_usd": {"input": "0.000001"}, "available_balance_usd": "2",
}}}, stream)
with open(aa_path, "w", encoding="utf-8") as stream:
json.dump({
"metric_id": catalog.AA_METRIC, "version": "1.5", "ingestion": "manual_audited",
"terms_accepted": True, "source_url": catalog.AA_URL,
"attribution": "Artificial Analysis Coding Agent Index", "captured_at": timestamp,
"records": [{
"provider": "openai", "model_id": "gpt-6-luna", "variant": "high", "score": "54",
"agent": "Codex", "harness": "OMP", "settings": {"effort": "high"},
"match_grade": "exact", "evidence_url": catalog.AA_URL,
}, {
"provider": "openai", "model_id": "gpt-6-luna", "variant": "xhigh", "score": "60",
"agent": "Codex", "harness": "OMP", "settings": {"effort": "xhigh"},
"match_grade": "exact", "evidence_url": catalog.AA_URL,
}],
}, stream)
rows = catalog.parse_omp_models({"models": [{"provider": "openai", "id": "gpt-6-luna",
"thinking": ["high"]}]}, "openai")
discover = lambda provider: {"ok": True, "models": ["openai/gpt-6-luna", "openai/gpt-6-luna#high"], "rows": rows,
"source_hash": "a" * 64, "verified_at": timestamp, "reason": None}
fetched = []
def fetcher(url):
fetched.append(url)
payload = {"data": []}
return payload, url, catalog.sha256(catalog.canonical_json(payload))
result = catalog.sync("omp", ["openai"], cache, fetcher, aa_snapshot=aa_path,
availability_snapshot=availability_path, price_overrides=native_path,
omp_discoverer=discover)
snapshot = catalog.load_snapshot("omp", cache)
by_key = {row["model_key"]: row for row in snapshot["models"]}
self.assertNotIn("openai/gpt-6-luna#xhigh", by_key)
self.assertEqual(result["model_count"], 2)
self.assertEqual(fetched, [catalog.OPENROUTER_URL])
model = by_key["openai/gpt-6-luna#high"]
self.assertEqual(model["score"]["version"], "1.5")
self.assertEqual(model["billing_mode"], "omp_payg")
self.assertTrue(model["availability"]["local_listed"])
self.assertTrue(model["availability"]["entitled"])
self.assertTrue(model["availability"]["balance_confirmed"])
self.assertEqual(model["native_tariff"]["available_balance_usd"], "2")
with self.assertRaisesRegex(ValueError, "providers explicitamente"):
catalog.sync("omp", cache_dir=cache, fetcher=lambda url: ({}, url, "x"))
with self.assertRaises(ValueError):
catalog.validate_native_tariff_overrides({"openai/model": {
"billing_mode": "omp_payg", "account_confirmed": True,
"source_url": "https://provider.example/pricing", "captured_at": "2026-09-25T12:00:00Z",
"price_per_token_usd": {"input": "0.1"},
}})
def test_manual_availability_and_native_tariff_are_auditable_without_secrets(self):
with tempfile.TemporaryDirectory() as cache:
availability = os.path.join(cache, "availability.json")
with open(availability, "w", encoding="utf-8") as stream:
json.dump({
"tool": "opencode", "ingestion": "manual_audited", "user_confirmed": True,
"captured_at": "2026-09-25T00:00:00Z",
"records": [{"provider": "opencode", "model_id": "model-a", "variant": None,
"local_listed": True, "authenticated": True, "entitled": True,
"verified_at": "2026-09-25T00:00:00Z", "source": "manual confirmation token=NEVER_PERSIST"}]
}, stream)
def fetcher(url):
body = {"data": [{"id": "model-a"}]} if url == catalog.ZEN_URL else {"data": []}
return body, url, catalog.sha256(catalog.canonical_json(body))
catalog.sync("opencode", ("opencode",), cache, fetcher,
lambda provider: {"ok": True, "models": ["model-a"], "reason": None},
availability_snapshot=availability)
snapshot = catalog.load_snapshot(cache_dir=cache)
model = snapshot["models"][0]
self.assertEqual(model["availability"]["authenticated"], True)
self.assertEqual(model["availability"]["entitled"], True)
self.assertEqual(model["availability"]["source"], "manual_audited")
self.assertNotIn("NEVER_PERSIST", json.dumps(snapshot))
self.assertIn("availability", snapshot["source_hashes"])
tariff = catalog.validate_native_tariff_overrides({"opencode/model-a": {
"billing_mode": "zen_balance", "account_confirmed": True, "source_url": "https://opencode.ai/docs/zen",
"captured_at": "2026-09-25T00:00:00Z", "price_per_token_usd": {"input": "0.000001", "output": "0.000002"},
"available_balance_usd": "2",
}})
self.assertEqual(tariff["opencode/model-a"]["price_per_token_usd"]["input"], "0.000001")
self.assertEqual(tariff["opencode/model-a"]["available_balance_usd"], "2")
with self.assertRaisesRegex(ValueError, "saldo disponível"):
catalog.validate_native_tariff_overrides({"opencode/model-a": {
"billing_mode": "zen_balance", "account_confirmed": True, "source_url": "https://opencode.ai/docs/zen",
"captured_at": "2026-09-25T00:00:00Z", "price_per_token_usd": {"input": "0.000001"},
}})
def test_unknown_openrouter_id_override_is_rejected(self):
payload = {"data": [{"id": "known/id", "pricing": {}}]}
with self.assertRaises(ValueError):
catalog.parse_openrouter(payload, {"typo/id": {"context_length": 100}})
class CatalogSnapshotTests(unittest.TestCase):
def test_missing_cache_and_complete_report_keep_unavailable_models(self):
with tempfile.TemporaryDirectory() as cache:
self.assertEqual(catalog.list_models(cache_dir=cache), [])
self.assertEqual(catalog._csv_text([]).splitlines()[0].split(","), list(catalog.LIST_FIELDS))
source_catalogs = {
catalog.ZEN_URL: {"data": [{"id": "glm-5.3", "name": "GLM"}, {"id": "unknown-model", "name": "Unknown"}]},
catalog.GO_URL: {"data": [{"id": "gpt-6-luna", "name": "Luna"}]},
catalog.OPENROUTER_URL: {"data": [{"id": "z-ai/glm-5.3", "pricing": {"prompt": "0.0000014", "completion": "0.0000044"}},
{"id": "openai/gpt-6-luna", "pricing": {"prompt": "0.0000001", "completion": "0.0000005"}}]},
}
def fetcher(url):
body = source_catalogs[url]
return body, url, catalog.sha256(catalog.canonical_json(body))
def local(provider):
ids = ["glm-5.3"] if provider == "opencode" else []
return {"ok": True, "models": ids, "reason": None}
result = catalog.sync("opencode", ("opencode", "opencode-go"), cache, fetcher, local)
rows = catalog.list_models(cache_dir=cache)
self.assertEqual(result["model_count"], 3)
self.assertEqual(len(rows), 3)
by_key = {row["model_key"]: row for row in rows}
self.assertEqual(by_key["opencode/glm-5.3"]["disponivel_na_conta"], "n/d")
self.assertEqual(by_key["opencode/unknown-model"]["openrouter_id"], "n/d")
self.assertEqual(by_key["opencode/unknown-model"]["preco_or_entrada_usd_1m"], "n/d")
self.assertEqual(by_key["opencode-go/gpt-6-luna"]["coding_index_nota"], "n/d")
self.assertFalse(by_key["opencode-go/gpt-6-luna"]["disponivel_na_conta"])
self.assertEqual(len(catalog.list_models(cache_dir=cache, show_missing=False)), 1)
self.assertEqual(Decimal(by_key["opencode/glm-5.3"]["preco_or_entrada_usd_1m"]), Decimal("1.4"))
def test_archived_cli_csv_reproduces_frozen_bytes_at_capture_time(self):
archived_snapshot = os.path.join(ROOT, "docs", "model-catalog-snapshot-2026-09-25", "catalog-opencode.json")
archived_csv = os.path.join(ROOT, "docs", "model-catalog-zen-go-2026-09-25.csv")
with tempfile.TemporaryDirectory() as cache:
shutil.copyfile(archived_snapshot, os.path.join(cache, "catalog-opencode.json"))
result = subprocess.run([
sys.executable, os.path.join(ROOT, "scripts", "model_catalog.py"), "list",
"--tool", "opencode", "--format", "csv", "--cache-dir", cache,
"--as-of-utc", "2026-09-25T07:18:38Z",
], capture_output=True, check=True)
with open(archived_csv, "rb") as stream:
expected = stream.read()
self.assertEqual(hashlib.sha256(expected).hexdigest(),
"80e2930877b20e5c47d4fd18f394c17a4683b257ca715496e91c63520d528230")
self.assertEqual(result.stdout, expected)
self.assertEqual(len(result.stdout.splitlines()) - 1, 122)
def test_archived_openrouter_override_changes_long_context_cost(self):
cache = os.path.join(ROOT, "docs", "model-catalog-snapshot-2026-09-25")
record = catalog.load_snapshot(cache_dir=cache)["openrouter"]["openai/gpt-6-sol"]
below = catalog.openrouter_price_for_profile(record, {"input": 271999})
above = catalog.openrouter_price_for_profile(record, {"input": 300000})
self.assertEqual(catalog.reference_cost(below["price_per_token_usd"], {"input": 271999}),
Decimal("0.543998"))
self.assertEqual(above["applied_context_tier"], 272000)
self.assertEqual(catalog.reference_cost(above["price_per_token_usd"], {"input": 300000}),
Decimal("1.2"))
def test_go_report_shows_all_confirmed_quota_windows_and_fallback_policy(self):
with tempfile.TemporaryDirectory() as cache:
timestamp = catalog.utc_now()
native_path = os.path.join(cache, "quota.json")
with open(native_path, "w", encoding="utf-8") as stream:
json.dump({"native_tariffs": {"opencode-go/model-a": {
"billing_mode": "go_quota", "account_confirmed": True,
"source_url": "https://opencode.ai/docs/go", "captured_at": timestamp,
"quota_window_remaining": {"5h": "2", "weekly": "5", "monthly": "10"},
"quota_window_debit": {"5h": "0.5", "weekly": "0.5", "monthly": "0.5"},
"use_balance_fallback": False,
}}}, stream)
def fetcher(url):
body = {"data": [{"id": "model-a"}]} if url == catalog.GO_URL else {"data": []}
return body, url, catalog.sha256(catalog.canonical_json(body))
catalog.sync("opencode", ("opencode-go",), cache, fetcher,
lambda _: {"ok": True, "models": ["model-a"], "reason": None},
price_overrides=native_path)
report = catalog.list_models(cache_dir=cache)[0]["preco_native_ou_cota"]
self.assertEqual(report["quota_window_remaining"], {"5h": "2", "weekly": "5", "monthly": "10"})
self.assertEqual(report["quota_window_debit"]["monthly"], "0.5")
self.assertFalse(report["use_balance_fallback"])
def test_score_bands_are_fixed_inclusive_only_at_high_upper_bound(self):
cases = {"9.99": None, "10": "leve", "19.99": "leve", "20": "media", "49.99": "media", "50": "alta", "100": "alta", "100.01": None}
for score, expected in cases.items():
self.assertEqual(catalog.band_for_score(score), expected)
if __name__ == "__main__":
unittest.main(verbosity=1)
+260
View File
@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""Behavior tests for role routing, spending boundaries, and deterministic ties."""
import os
import json
import sys
import tempfile
import unittest
from datetime import datetime, timezone
from decimal import Decimal
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "scripts"))
import model_catalog
import model_router
NOW = datetime(2026, 9, 25, 12, 0, tzinfo=timezone.utc)
NOW_TEXT = "2026-09-25T12:00:00Z"
def candidate(model_key, *, provider="opencode", score="45", native_input="0.0000001", billing="zen_balance", **extra):
result = {
"tool": "opencode", "model_key": model_key, "provider": provider,
"model_id": model_key.split("/", 1)[-1], "variant": None, "billing_mode": billing,
"availability": {"local_listed": True, "authenticated": True, "entitled": True, "verified_at": NOW_TEXT,
"local_verified_at": NOW_TEXT, "account_verified_at": NOW_TEXT,
"balance_confirmed": True, "balance_verified_at": NOW_TEXT},
"score": {"metric_id": model_catalog.AA_METRIC, "version": "1.5", "score": score,
"agent": "Codex", "harness": "OpenCode", "variant": None, "settings": {"effort": "max"},
"match_grade": "exact", "source_url": model_catalog.AA_URL, "observed_at": NOW_TEXT},
"openrouter": {"captured_at": NOW_TEXT, "price_per_token_usd": {"input": "0.0000001", "output": "0.0000005"}},
"native_tariff": {"billing_mode": billing, "account_confirmed": True, "source_url": "https://opencode.ai/docs/zen",
"captured_at": NOW_TEXT, "available_balance_usd": "2",
"price_per_token_usd": {"input": native_input, "output": "0.0000002",
"cache_read": "0.00000001", "cache_write": "0.00000002"}},
}
if billing == "go_quota":
result["quota_verified_at"] = NOW_TEXT
result["quota_source_url"] = "https://opencode.ai/docs/go"
result["use_balance_fallback"] = False
result.update(extra)
return result
class RoutePolicyTests(unittest.TestCase):
def test_missing_cache_is_explicit_and_no_model_is_invented(self):
with tempfile.TemporaryDirectory() as cache:
result = model_router.route_from_catalog("opencode", "implementer", ["opencode"], cache)
self.assertIsNone(result["selected"])
self.assertIn("snapshot_de_catalogo_ausente", result["reason"])
empty = model_router.route([], "explorer", allowed_providers=["opencode"], now=NOW)
self.assertIsNone(empty["selected"])
self.assertEqual(empty["reason"], "no_scored_candidate")
high_only = model_router.route([candidate("opencode/high-only", score="54")], "implementer",
allowed_providers=["opencode"], budget_usd="1", now=NOW)
self.assertIsNone(high_only["selected"])
self.assertEqual(high_only["reason"], "faixa_vazia:media")
def test_benchmark_with_wrong_index_is_never_ranked(self):
row = candidate("opencode/model-a", score="54")
row["score"]["metric_id"] = "artificial_analysis_intelligence_index"
result = model_router.route([row], "planner", allowed_providers=["opencode"], now=NOW)
self.assertIsNone(result["selected"])
self.assertEqual(result["ranking"][0]["eligible"], False)
self.assertIn("indice_incompativel_ou_ausente", result["ranking"][0]["exclusion_reasons"])
def test_role_band_precedes_cost_and_plateau_uses_native_cost_not_openrouter(self):
cheap_high = candidate("opencode/high", score="54", native_input="0.000000001")
media_best = candidate("opencode/media", score="49", native_input="0.0000001")
media_cheaper = candidate("opencode/media-cheap", score="47", native_input="0.00000005")
result = model_router.route([cheap_high, media_best, media_cheaper], "implementer", allowed_providers=["opencode"],
tokens={"input": 1_000_000}, budget_usd="0.08", now=NOW)
self.assertEqual(result["selected"]["model_key"], "opencode/media-cheap")
self.assertEqual(result["selected"]["band"], "media")
self.assertEqual(Decimal(result["selected"]["reference_cost_usd"]), Decimal("0.1"))
self.assertEqual(Decimal(result["selected"]["estimated_native_usd"]), Decimal("0.05"))
self.assertIn("platô", result["reason"])
def test_zen_balance_must_cover_native_estimate_and_be_confirmed(self):
row = candidate("opencode/model-a")
row["native_tariff"]["available_balance_usd"] = "0.05"
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 1_000_000}, budget_usd="1", now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("saldo_insuficiente", result["ranking"][0]["exclusion_reasons"])
row["native_tariff"]["available_balance_usd"] = "1"
row["availability"]["balance_confirmed"] = False
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 1_000_000}, budget_usd="1", now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("saldo_nao_confirmado_ou_obsoleto", result["ranking"][0]["exclusion_reasons"])
def test_zen_balance_requires_explicit_budget(self):
row = candidate("opencode/model-a")
result = model_router.route([row], "implementer", allowed_providers=["opencode"], now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("orcamento_zen_nao_configurado", result["ranking"][0]["exclusion_reasons"])
def test_exact_benchmark_harness_must_match_tool_but_proxy_requires_approval(self):
row = candidate("opencode/model-a")
row["score"]["harness"] = "Codex"
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 100_000}, budget_usd="1", now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("harness_incompativel", result["ranking"][0]["exclusion_reasons"])
row["score"]["match_grade"] = "proxy"
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 100_000}, budget_usd="1", now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("match_proxy_nao_autorizado", result["ranking"][0]["exclusion_reasons"])
approved = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 100_000}, budget_usd="1", allow_proxy=True, now=NOW)
self.assertEqual(approved["selected"]["model_key"], "opencode/model-a")
def test_budget_is_applied_before_plateau_and_proxy_requires_explicit_permission(self):
top = candidate("opencode/top", score="49", native_input="0.000001")
lower = candidate("opencode/lower", score="46", native_input="0.00000005")
row = candidate("opencode/proxy", score="45", match="proxy")
row["score"]["match_grade"] = "proxy"
result = model_router.route([top, lower, row], "implementer", allowed_providers=["opencode"],
tokens={"input": 1_000_000}, budget_usd="0.1", now=NOW)
self.assertEqual(result["selected"]["model_key"], "opencode/lower")
exclusions = {entry["model_key"]: entry["exclusion_reasons"] for entry in result["ranking"]}
self.assertIn("match_proxy_nao_autorizado", exclusions["opencode/proxy"])
proxy = model_router.route([row], "implementer", allowed_providers=["opencode"], tokens={"input": 1_000_000},
budget_usd="1", allow_proxy=True, now=NOW)
self.assertEqual(proxy["selected"]["model_key"], "opencode/proxy")
def test_unknown_entitlement_and_provider_are_not_eligible(self):
row = candidate("opencode/model-a", score="54")
row["availability"]["entitled"] = None
result = model_router.route([row], "planner", allowed_providers=["opencode"], now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("entitlement_nao_confirmado", result["ranking"][0]["exclusion_reasons"])
unauthorized = model_router.route([candidate("opencode/model-a", score="54")], "planner", now=NOW)
self.assertIn("provider_nao_autorizado", unauthorized["reason"])
def test_go_quota_checks_every_window_and_never_falls_back_to_zen(self):
for window in model_router.WINDOWS:
row = candidate("opencode-go/model-a", provider="opencode-go", score="54", billing="go_quota")
row["quota_window_remaining"] = {"5h": "1", "weekly": "1", "monthly": "1"}
row["quota_window_debit"] = {"5h": "0.1", "weekly": "0.1", "monthly": "0.1"}
row["quota_window_remaining"][window] = "0.05"
result = model_router.route([row], "planner", allowed_providers=["opencode-go"], tokens={"input": 1_000_000}, now=NOW)
self.assertIsNone(result["selected"], window)
self.assertIn("quota_go_insuficiente", result["reason"])
row = candidate("opencode-go/model-a", provider="opencode-go", score="54", billing="go_quota")
row["quota_window_remaining"] = {"5h": "1", "weekly": "1", "monthly": "1"}
row["quota_window_debit"] = {"5h": "0.1", "weekly": "0.1", "monthly": "0.1"}
row["use_balance_fallback"] = True
result = model_router.route([row], "planner", allowed_providers=["opencode-go"], tokens={"input": 1_000_000}, now=NOW)
self.assertIsNone(result["selected"])
self.assertEqual(result["reason"], "fallback_go_para_saldo_zen_nao_autorizado")
def test_go_success_exposes_quota_and_billing_mode_separately(self):
row = candidate("opencode-go/model-a", provider="opencode-go", score="54", billing="go_quota")
row["quota_window_remaining"] = {"5h": "1", "weekly": "2", "monthly": "3"}
row["quota_window_debit"] = {"5h": "0.1", "weekly": "0.1", "monthly": "0.2"}
result = model_router.route([row], "planner", allowed_providers=["opencode-go"], tokens={"input": 1_000_000}, now=NOW)
selected = result["selected"]
self.assertEqual(selected["billing_mode"], "go")
self.assertEqual(selected["billing_mode_detail"], "go_quota")
self.assertEqual(selected["quota_window_debit"]["monthly"], "0.2")
self.assertIsNone(selected["estimated_native_usd"])
self.assertEqual(selected["native_cost_or_quota_debit_usd"], "0.2")
def test_missing_cache_rate_disables_comparative_cost_not_native_selection(self):
row = candidate("opencode/model-a", score="45")
row["openrouter"]["price_per_token_usd"] = {"input": "0.0000001"}
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
tokens={"input": 1_000_000, "cache_read": 1000}, budget_usd="1", now=NOW)
self.assertEqual(result["selected"]["model_key"], "opencode/model-a")
self.assertIsNone(result["selected"]["reference_cost_usd"])
self.assertEqual(Decimal(result["selected"]["estimated_native_usd"]), Decimal("0.10001"))
def test_expired_paid_metadata_blocks_automatic_selection(self):
row = candidate("opencode/model-a", score="54")
row["openrouter"]["captured_at"] = "2026-09-23T00:00:00Z"
result = model_router.route([row], "planner", allowed_providers=["opencode"], budget_usd="1", now=NOW)
self.assertIsNone(result["selected"])
self.assertIn("preco_referencial_obsoleto_ou_ausente", result["ranking"][0]["exclusion_reasons"])
def test_ties_are_deterministic_independent_of_input_order(self):
a = candidate("opencode/a", score="48", native_input="0.00000005")
b = candidate("opencode/b", score="48", native_input="0.00000005")
first = model_router.route([a, b], "implementer", allowed_providers=["opencode"], tokens={"input": 1_000_000}, budget_usd="1", now=NOW)
second = model_router.route([b, a], "implementer", allowed_providers=["opencode"], tokens={"input": 1_000_000}, budget_usd="1", now=NOW)
self.assertEqual(first, second)
self.assertEqual(first["selected"]["model_key"], "opencode/a")
def test_fixed_bands_metric_and_role_override_contract(self):
self.assertEqual(model_router.validate_bands({"leve": [10, 20], "media": [20, 50], "alta": [50, 100]}),
model_router.FIXED_BANDS)
with self.assertRaises(ValueError):
model_router.validate_bands({"leve": [0, 20], "media": [20, 50], "alta": [50, 100]})
row = candidate("opencode/model-a", score="54")
result = model_router.route([row], "implementer", allowed_providers=["opencode"],
band_override="alta", tokens={"input": 1_000_000}, budget_usd="1", now=NOW)
self.assertEqual(result["selected"]["band"], "alta")
with self.assertRaises(ValueError):
model_router.route([row], "planner", allowed_providers=["opencode"],
benchmark_metric="aa_coding_agent_index_v1.4", now=NOW)
def test_omp_payg_uses_only_native_estimate_and_exact_dispatch_variant(self):
row = candidate("openai/model-a#high", provider="openai", score="54", billing="omp_payg")
row.update({"tool": "omp", "model_id": "model-a", "variant": "high"})
row["score"]["harness"] = "OMP"
row["score"]["variant"] = "high"
row["openrouter"] = None
row["native_tariff"] = {
"billing_mode": "omp_payg", "account_confirmed": True,
"source_url": "https://provider.example/pricing", "captured_at": NOW_TEXT,
"price_per_token_usd": {"input": "0.000001"}, "available_balance_usd": "0.2",
}
result = model_router.route([row], "planner", allowed_providers=["openai"],
tokens={"input": 100_000}, budget_usd="0.2", now=NOW)
selected = result["selected"]
self.assertEqual(selected["model_key"], "openai/model-a#high")
self.assertEqual(selected["dispatch_model_key"], "openai/model-a:high")
self.assertEqual(selected["billing_mode"], "payg")
self.assertEqual(selected["estimated_native_usd"], "0.100000")
self.assertEqual(selected["available_balance_usd"], "0.2")
self.assertIsNone(selected["reference_cost_usd"])
row["availability"]["balance_confirmed"] = False
blocked = model_router.route([row], "planner", allowed_providers=["openai"],
tokens={"input": 100_000}, budget_usd="0.2", now=NOW)
self.assertIsNone(blocked["selected"])
self.assertIn("saldo_nao_confirmado_ou_obsoleto", blocked["ranking"][0]["exclusion_reasons"])
def test_omp_catalog_route_revalidates_local_list_and_keeps_manual_entitlement(self):
row = candidate("openai/model-a#high", provider="openai", score="54", billing="omp_payg")
row.update({"tool": "omp", "model_id": "model-a", "variant": "high"})
row["score"]["harness"] = "OMP"
row["score"]["variant"] = "high"
row["native_tariff"] = {
"billing_mode": "omp_payg", "account_confirmed": True,
"source_url": "https://provider.example/pricing", "captured_at": NOW_TEXT,
"price_per_token_usd": {"input": "0.000001"}, "available_balance_usd": "0.2",
}
with tempfile.TemporaryDirectory() as cache:
with open(os.path.join(cache, "catalog-omp.json"), "w", encoding="utf-8") as stream:
json.dump({"tool": "omp", "models": [row]}, stream)
result = model_router.route_from_catalog(
"omp", "planner", ["openai"], cache, tokens={"input": 100_000}, budget_usd="0.2",
omp_discoverer=lambda provider: {"ok": True, "models": ["openai/model-a#high"], "verified_at": NOW_TEXT},
now=NOW,
)
unsupported = model_router.route_from_catalog(
"omp", "planner", ["openai"], cache, tokens={"input": 100_000}, budget_usd="0.2",
omp_discoverer=lambda provider: {"ok": True, "models": ["openai/model-a"], "verified_at": NOW_TEXT},
now=NOW,
)
self.assertIsNone(unsupported["selected"])
self.assertIn("nao_listado_localmente", unsupported["ranking"][0]["exclusion_reasons"])
self.assertEqual(result["selected"]["dispatch_model_key"], "openai/model-a:high")
if __name__ == "__main__":
unittest.main(verbosity=1)