refactor(catalog): split validators and staleness checks below CC 10, with path tests

Adds qa/test_branch_paths.py pinning the main path and one inverted path per
decision for validate_aa_snapshot, validate_availability_snapshot,
validate_native_tariff_overrides, model_router._staleness and
_native_cost_and_headroom, then splits each into small helpers. Messages,
check order and results are unchanged.
This commit is contained in:
evandro committed 2026-10-10 04:05:58 -03:00
1 parent fda614e2e7
commit 1aa2c68818
3 files changed
+785 -199

No files matched your search

+185 -111
View File
@@ -338,72 +338,98 @@ def reference_cost(price_per_token_usd: dict[str, Any], tokens: dict[str, Any])
return total
def validate_aa_snapshot(payload: Any) -> list[dict[str, Any]]:
"""Aceita exclusivamente export manual auditável do Coding Agent Index v1.5."""
if not isinstance(payload, dict):
raise ValueError("snapshot AA deve ser objeto")
allowed_root = {"metric_id", "version", "ingestion", "terms_accepted", "source_url", "attribution", "captured_at", "records"}
if set(payload) - allowed_root:
raise ValueError("snapshot AA contém campo não permitido")
if payload.get("metric_id") != AA_METRIC or str(payload.get("version")) != AA_VERSION:
raise ValueError("somente aa_coding_agent_index v1.5 é aceito; índices não são intercambiáveis")
_AA_ROOT_FIELDS = {"metric_id", "version", "ingestion", "terms_accepted", "source_url", "attribution", "captured_at", "records"}
_AA_RECORD_FIELDS = {"metric_id", "version", "provider", "model_id", "variant", "score", "agent", "harness",
"settings", "match_grade", "evidence_url", "components", "fallback", "effort", "reasoning", "configuration"}
_AA_PROVENANCE = ("agent", "harness", "settings", "evidence_url")
def _check_aa_provenance(payload: dict[str, Any]) -> None:
"""Consentimento, fonte oficial, atribuição e data de captura do export AA."""
if payload.get("ingestion") != "manual_audited" or payload.get("terms_accepted") is not True:
raise ValueError("snapshot precisa ser importação manual consentida (terms_accepted=true)")
source_url = payload.get("source_url")
if not _secure_url(source_url, "artificialanalysis.ai"):
if not _secure_url(payload.get("source_url"), "artificialanalysis.ai"):
raise ValueError("source_url oficial da Artificial Analysis é obrigatório")
if not isinstance(payload.get("attribution"), str) or not payload["attribution"]:
raise ValueError("atribuição AA obrigatória")
captured_at = payload.get("captured_at")
captured_time = _parse_utc(captured_at)
captured_time = _parse_utc(payload.get("captured_at"))
if captured_time is None or captured_time > dt.datetime.now(dt.timezone.utc):
raise ValueError("captured_at obrigatório, UTC/offset-aware e não futuro")
def _aa_records(payload: Any) -> list[Any]:
"""Valida o cabeçalho do export AA (índice e proveniência) e devolve os registros."""
if not isinstance(payload, dict):
raise ValueError("snapshot AA deve ser objeto")
if set(payload) - _AA_ROOT_FIELDS:
raise ValueError("snapshot AA contém campo não permitido")
if payload.get("metric_id") != AA_METRIC or str(payload.get("version")) != AA_VERSION:
raise ValueError("somente aa_coding_agent_index v1.5 é aceito; índices não são intercambiáveis")
_check_aa_provenance(payload)
records = payload.get("records")
if not isinstance(records, list):
raise ValueError("records deve ser lista")
return records
def _aa_record_key(record: Any) -> tuple[str, str, str | None]:
"""Valida a estrutura e o mapeamento exato de um registro AA; devolve (provider, model_id, variant)."""
if not isinstance(record, dict) or set(record) - _AA_RECORD_FIELDS:
raise ValueError("registro AA inválido ou contém campo não permitido")
if record.get("metric_id", AA_METRIC) != AA_METRIC or str(record.get("version", AA_VERSION)) != AA_VERSION:
raise ValueError("índice/versão diferente não pode ser misturado")
provider, model_id = record.get("provider"), record.get("model_id")
if not _is_exact_model_ref(provider, model_id):
raise ValueError("mapeamento provider/model_id exato obrigatório")
variant = record.get("variant")
if variant is not None and (not isinstance(variant, str) or not variant):
raise ValueError("variante AA inválida")
return provider, model_id, variant
def _aa_record_score(record: dict[str, Any]) -> Decimal:
"""Valida nota, grau de correspondência e proveniência de um registro AA; devolve a nota."""
score = decimal_value(record.get("score"))
if score is None or score < 0 or score > 100:
raise ValueError("score AA precisa estar no intervalo 0..100")
if record.get("match_grade") not in ("exact", "proxy"):
raise ValueError("match_grade deve ser exact ou proxy, sem fuzzy match")
for field in _AA_PROVENANCE:
if field not in record or record[field] in (None, ""):
raise ValueError(f"campo de proveniência AA obrigatório: {field}")
if not _secure_url(record["evidence_url"], "artificialanalysis.ai"):
raise ValueError("evidence_url AA inválida")
return score
def validate_aa_snapshot(payload: Any) -> list[dict[str, Any]]:
"""Aceita exclusivamente export manual auditável do Coding Agent Index v1.5."""
records = _aa_records(payload)
validated = []
seen_keys: set[tuple[str, str, str | None]] = set()
allowed_record = {"metric_id", "version", "provider", "model_id", "variant", "score", "agent", "harness",
"settings", "match_grade", "evidence_url", "components", "fallback", "effort", "reasoning", "configuration"}
for record in records:
if not isinstance(record, dict) or set(record) - allowed_record:
raise ValueError("registro AA inválido ou contém campo não permitido")
if record.get("metric_id", AA_METRIC) != AA_METRIC or str(record.get("version", AA_VERSION)) != AA_VERSION:
raise ValueError("índice/versão diferente não pode ser misturado")
provider, model_id = record.get("provider"), record.get("model_id")
if not isinstance(provider, str) or not isinstance(model_id, str) or not model_id or "/" in model_id or "#" in model_id:
raise ValueError("mapeamento provider/model_id exato obrigatório")
variant = record.get("variant")
if variant is not None and (not isinstance(variant, str) or not variant):
raise ValueError("variante AA inválida")
key = (provider, model_id, variant)
key = _aa_record_key(record)
if key in seen_keys:
raise ValueError("benchmark inconsistente: modelo/variante duplicado no snapshot")
seen_keys.add(key)
score = decimal_value(record.get("score"))
if score is None or score < 0 or score > 100:
raise ValueError("score AA precisa estar no intervalo 0..100")
grade = record.get("match_grade")
if grade not in ("exact", "proxy"):
raise ValueError("match_grade deve ser exact ou proxy, sem fuzzy match")
for key in ("agent", "harness", "settings", "evidence_url"):
if key not in record or record[key] in (None, ""):
raise ValueError(f"campo de proveniência AA obrigatório: {key}")
if not _secure_url(record["evidence_url"], "artificialanalysis.ai"):
raise ValueError("evidence_url AA inválida")
score = _aa_record_score(record)
validated.append({
**record, "metric_id": AA_METRIC, "version": AA_VERSION, "score": decimal_text(score),
"source_url": source_url, "attribution": payload["attribution"], "observed_at": captured_at,
"source_url": payload["source_url"], "attribution": payload["attribution"], "observed_at": payload["captured_at"],
"snapshot_hash": sha256(canonical_json(payload)),
})
return validated
def validate_availability_snapshot(payload: Any) -> dict[tuple[str, str, str | None], dict[str, Any]]:
"""Validates explicit account evidence without accepting or retaining secrets."""
_AVAILABILITY_FIELDS = {"provider", "model_id", "variant", "local_listed", "authenticated", "entitled",
"verified_at", "source", "balance_confirmed", "balance_verified_at"}
_AVAILABILITY_REQUIRED = ("local_listed", "authenticated", "entitled", "verified_at", "source")
def _availability_records(payload: Any) -> list[Any]:
"""Valida o cabeçalho do snapshot de disponibilidade e devolve a lista de registros."""
if not isinstance(payload, dict) or payload.get("tool") not in ("opencode", "omp") or payload.get("ingestion") != "manual_audited":
raise ValueError("availability precisa ser snapshot manual auditado de opencode ou omp")
tool = payload["tool"]
if payload.get("user_confirmed") is not True:
raise ValueError("snapshot de disponibilidade exige user_confirmed=true")
captured_at = payload.get("captured_at")
@@ -418,100 +444,148 @@ def validate_availability_snapshot(payload: Any) -> dict[tuple[str, str, str | N
records = payload.get("records")
if not isinstance(records, list):
raise ValueError("records de disponibilidade deve ser lista")
return records
def _is_exact_model_ref(provider: Any, model_id: Any) -> bool:
return isinstance(provider, str) and isinstance(model_id, str) and bool(model_id) and "/" not in model_id and "#" not in model_id
def _availability_key(record: Any) -> tuple[str, str, str | None]:
"""Valida a estrutura de um registro e devolve a identidade (provider, model_id, variant)."""
if not isinstance(record, dict) or set(record) - _AVAILABILITY_FIELDS:
raise ValueError("snapshot de disponibilidade contém campos não permitidos")
provider, model_id, variant = record.get("provider"), record.get("model_id"), record.get("variant")
if not _is_exact_model_ref(provider, model_id) or (variant is not None and not isinstance(variant, str)):
raise ValueError("provider/model_id/variant exatos são obrigatórios")
if any(field not in record for field in _AVAILABILITY_REQUIRED):
raise ValueError("snapshot deve declarar todos os campos de disponibilidade")
return provider, model_id, variant
def _check_availability_balance(record: dict[str, Any], now: dt.datetime) -> None:
balance_confirmed = record.get("balance_confirmed")
if balance_confirmed is not None and not isinstance(balance_confirmed, bool):
raise ValueError("balance_confirmed deve ser bool ou null")
balance_verified_at = record.get("balance_verified_at")
balance_time = _parse_utc(balance_verified_at) if balance_verified_at else None
if balance_time is not None and (balance_time > now or balance_confirmed is not True):
raise ValueError("balance_verified_at requer confirmação explícita e não pode ser futuro")
if balance_confirmed is True and balance_time is None:
raise ValueError("saldo confirmado exige balance_verified_at UTC válido")
def _check_availability_evidence(record: dict[str, Any]) -> None:
"""Valida flags booleanas, saldo confirmado e a evidência (verified_at/source) de um registro."""
now = dt.datetime.now(dt.timezone.utc)
for field in ("local_listed", "authenticated", "entitled"):
if record[field] is not None and not isinstance(record[field], bool):
raise ValueError(f"{field} deve ser bool ou null")
_check_availability_balance(record, now)
verified_at = _parse_utc(record["verified_at"])
if verified_at is None or verified_at > now or not isinstance(record["source"], str) or not record["source"]:
raise ValueError("source e verified_at UTC válido são necessários para auditar a disponibilidade")
def validate_availability_snapshot(payload: Any) -> dict[tuple[str, str, str | None], dict[str, Any]]:
"""Validates explicit account evidence without accepting or retaining secrets."""
records = _availability_records(payload)
result = {}
source_hash = sha256(canonical_json(payload))
allowed_fields = {"provider", "model_id", "variant", "local_listed", "authenticated", "entitled",
"verified_at", "source", "balance_confirmed", "balance_verified_at"}
for record in records:
if not isinstance(record, dict) or set(record) - allowed_fields:
raise ValueError("snapshot de disponibilidade contém campos não permitidos")
provider, model_id, variant = record.get("provider"), record.get("model_id"), record.get("variant")
if not isinstance(provider, str) or not isinstance(model_id, str) or not model_id or "/" in model_id or "#" in model_id or (variant is not None and not isinstance(variant, str)):
raise ValueError("provider/model_id/variant exatos são obrigatórios")
if any(field not in record for field in ("local_listed", "authenticated", "entitled", "verified_at", "source")):
raise ValueError("snapshot deve declarar todos os campos de disponibilidade")
key = (provider, model_id, variant)
key = _availability_key(record)
if key in result:
raise ValueError("snapshot de disponibilidade contém ID/variante duplicado")
for field in ("local_listed", "authenticated", "entitled"):
if record[field] is not None and not isinstance(record[field], bool):
raise ValueError(f"{field} deve ser bool ou null")
balance_confirmed = record.get("balance_confirmed")
if balance_confirmed is not None and not isinstance(balance_confirmed, bool):
raise ValueError("balance_confirmed deve ser bool ou null")
balance_verified_at = record.get("balance_verified_at")
balance_time = _parse_utc(balance_verified_at) if balance_verified_at else None
if balance_time is not None and (balance_time > dt.datetime.now(dt.timezone.utc) or balance_confirmed is not True):
raise ValueError("balance_verified_at requer confirmação explícita e não pode ser futuro")
if balance_confirmed is True and balance_time is None:
raise ValueError("saldo confirmado exige balance_verified_at UTC válido")
verified_at = _parse_utc(record["verified_at"])
if verified_at is None or verified_at > dt.datetime.now(dt.timezone.utc) or not isinstance(record["source"], str) or not record["source"]:
raise ValueError("source e verified_at UTC válido são necessários para auditar a disponibilidade")
_check_availability_evidence(record)
result[key] = {
"local_listed": record["local_listed"], "authenticated": record["authenticated"],
"entitled": record["entitled"], "verified_at": record["verified_at"],
"source": "manual_audited", "source_hash": source_hash,
"balance_confirmed": balance_confirmed, "balance_verified_at": balance_verified_at,
"balance_confirmed": record.get("balance_confirmed"), "balance_verified_at": record.get("balance_verified_at"),
}
return result
_NATIVE_ALLOWED = {"billing_mode", "price_per_token_usd", "quota_window_remaining", "quota_window_debit",
"use_balance_fallback", "source_url", "captured_at", "account_confirmed", "available_balance_usd"}
_NATIVE_BILLING_MODES = ("zen_balance", "go_quota", "omp_payg")
_NATIVE_RATE_KEYS = ("input", "output", "cache_read", "cache_write", "cache_write_5m", "cache_write_1h", "cache_write_24h")
_QUOTA_WINDOWS = ("5h", "weekly", "monthly")
def _check_native_header(model_key: Any, value: Any) -> str:
"""Valida estrutura, confirmação, modo, URL e data comuns a todo override; devolve o modo."""
if not isinstance(model_key, str) or not isinstance(value, dict) or set(value) - _NATIVE_ALLOWED:
raise ValueError("override native contém estrutura/campo inválido")
if value.get("account_confirmed") is not True:
raise ValueError("tarifa/cota native exige confirmação explícita da conta")
mode = value.get("billing_mode")
if mode not in _NATIVE_BILLING_MODES:
raise ValueError("billing_mode native inválido")
if not _secure_url(value.get("source_url")):
raise ValueError("source_url HTTPS de tarifa/cota native obrigatório")
try:
captured = dt.datetime.fromisoformat(value.get("captured_at").replace("Z", "+00:00"))
except (AttributeError, ValueError) as exc:
raise ValueError("captured_at de tarifa/cota native inválido") from exc
if captured.tzinfo is None or captured > dt.datetime.now(dt.timezone.utc):
raise ValueError("captured_at de tarifa/cota native precisa ser aware e não futuro")
return mode
def _non_negative_text(text: str | None) -> bool:
return text is not None and Decimal(text) >= 0
def _native_balance_fields(value: dict[str, Any]) -> dict[str, Any]:
"""Campos de tarifa por token e saldo (zen_balance e omp_payg)."""
rates = value.get("price_per_token_usd")
if not isinstance(rates, dict) or not rates:
raise ValueError("tarifa nativa precisa de price_per_token_usd")
prices = {k: decimal_text(decimal_value(v)) for k, v in rates.items() if k in _NATIVE_RATE_KEYS}
if len(prices) != len(rates) or not all(_non_negative_text(v) for v in prices.values()):
raise ValueError("componentes de tarifa nativa inválidos")
balance = decimal_value(value.get("available_balance_usd"))
if balance is None or balance < 0:
raise ValueError("Zen/OMP payg exige saldo disponível confirmado")
return {"price_per_token_usd": prices, "available_balance_usd": decimal_text(balance)}
def _quota_window_texts(source: dict[str, Any]) -> dict[str, str | None]:
return {k: decimal_text(decimal_value(source[k])) for k in _QUOTA_WINDOWS}
def _native_quota_fields(value: dict[str, Any]) -> dict[str, Any]:
"""Campos de cota por janela e fallback de saldo (go_quota)."""
remaining, debit = value.get("quota_window_remaining"), value.get("quota_window_debit")
if not isinstance(remaining, dict) or not isinstance(debit, dict) or not set(_QUOTA_WINDOWS) <= set(remaining) & set(debit):
raise ValueError("cota Go exige valores por modelo em 5h/weekly/monthly")
fields = {
"quota_window_remaining": _quota_window_texts(remaining),
"quota_window_debit": _quota_window_texts(debit),
}
if not all(_non_negative_text(v) for window_values in fields.values() for v in window_values.values()):
raise ValueError("quota Go precisa ter todos os valores decimais não negativos")
fallback = value.get("use_balance_fallback", False)
if not isinstance(fallback, bool):
raise ValueError("use_balance_fallback precisa ser bool")
fields["use_balance_fallback"] = fallback
return fields
def validate_native_tariff_overrides(payload: Any) -> dict[str, dict[str, Any]]:
"""Valida cobrança nativa/quotas manuais; preços OpenRouter nunca são tarifa."""
if not isinstance(payload, dict):
raise ValueError("native_tariffs deve ser objeto JSON")
result = {}
allowed = {"billing_mode", "price_per_token_usd", "quota_window_remaining", "quota_window_debit",
"use_balance_fallback", "source_url", "captured_at", "account_confirmed", "available_balance_usd"}
for model_key, value in payload.items():
if not isinstance(model_key, str) or not isinstance(value, dict) or set(value) - allowed:
raise ValueError("override native contém estrutura/campo inválido")
if value.get("account_confirmed") is not True:
raise ValueError("tarifa/cota native exige confirmação explícita da conta")
mode = value.get("billing_mode")
if mode not in ("zen_balance", "go_quota", "omp_payg"):
raise ValueError("billing_mode native inválido")
if not _secure_url(value.get("source_url")):
raise ValueError("source_url HTTPS de tarifa/cota native obrigatório")
captured_at = value.get("captured_at")
try:
captured = dt.datetime.fromisoformat(captured_at.replace("Z", "+00:00"))
except (AttributeError, ValueError) as exc:
raise ValueError("captured_at de tarifa/cota native inválido") from exc
if captured.tzinfo is None or captured > dt.datetime.now(dt.timezone.utc):
raise ValueError("captured_at de tarifa/cota native precisa ser aware e não futuro")
mode = _check_native_header(model_key, value)
record = {k: value[k] for k in ("billing_mode", "source_url", "captured_at", "account_confirmed")}
if mode in ("zen_balance", "omp_payg"):
rates = value.get("price_per_token_usd")
if not isinstance(rates, dict) or not rates:
raise ValueError("tarifa nativa precisa de price_per_token_usd")
record["price_per_token_usd"] = {
k: decimal_text(decimal_value(v)) for k, v in rates.items()
if k in ("input", "output", "cache_read", "cache_write", "cache_write_5m", "cache_write_1h", "cache_write_24h")
}
if len(record["price_per_token_usd"]) != len(rates) or any(v is None or Decimal(v) < 0 for v in record["price_per_token_usd"].values()):
raise ValueError("componentes de tarifa nativa inválidos")
balance = decimal_value(value.get("available_balance_usd"))
if balance is None or balance < 0:
raise ValueError("Zen/OMP payg exige saldo disponível confirmado")
record["available_balance_usd"] = decimal_text(balance)
else:
remaining, debit = value.get("quota_window_remaining"), value.get("quota_window_debit")
if not isinstance(remaining, dict) or not isinstance(debit, dict) or any(k not in remaining or k not in debit for k in ("5h", "weekly", "monthly")):
raise ValueError("cota Go exige valores por modelo em 5h/weekly/monthly")
record["quota_window_remaining"] = {k: decimal_text(decimal_value(remaining[k])) for k in ("5h", "weekly", "monthly")}
record["quota_window_debit"] = {k: decimal_text(decimal_value(debit[k])) for k in ("5h", "weekly", "monthly")}
if any(record[key][window] is None or Decimal(record[key][window]) < 0 for key in ("quota_window_remaining", "quota_window_debit") for window in ("5h", "weekly", "monthly")):
raise ValueError("quota Go precisa ter todos os valores decimais não negativos")
fallback = value.get("use_balance_fallback", False)
if not isinstance(fallback, bool):
raise ValueError("use_balance_fallback precisa ser bool")
record["use_balance_fallback"] = fallback
record.update(_native_quota_fields(value) if mode == "go_quota" else _native_balance_fields(value))
result[model_key] = record
return result
def discover_local_models(provider: str, executable: str = "opencode", runner: Callable[..., Any] = subprocess.run) -> dict[str, Any]:
"""Lista IDs pela CLI sem persistir stdout bruto, stderr, environment ou credenciais."""
try: