feat(proxy): implement metrics persistence, traffic logs DB, and advanced monitoring

- Add SQLite schema for proxy metrics and traffic logs
- Implement ProxyMetricsRepo with historical aggregates
- Add TrafficLogsRepo for DB-backed traffic log queries
- Create proxy-advanced IPC handlers (metrics, traffic logs, circuit breaker, parity)
- Add ProxyMetricsService with connection snapshots and cleanup
- Add ProxyReplayService and ProxyIdeConfigService
- Implement MonitorTab and ToolsTab for advanced proxy features
- Add TrafficLogTable with sorting and filtering
- Fix i18n keys for proxy metrics
- Fix generateIdeConfig type mapping
- Update proxy routes with advanced tab integration
This commit is contained in:
Gemini Nexus Bot committed 2026-04-23 00:57:01 -03:00
1 parent c73537d44c
commit 0be7d7fea2
36 files changed
+4551 -882

No files matched your search

@@ -6,6 +6,7 @@ import { GoogleAPIService } from '../../../services/GoogleAPIService';
import { getServerConfig } from '../../server-config';
import { RateLimitReason, RateLimitTracker } from './rate-limit-tracker';
import { updateDynamicForwardingRules } from '../../../lib/geminiNexus/ModelMapping';
import { registerProxyAdvancedService } from '../../../ipc/proxy-advanced/service-registry';
interface TokenData {
email: string;
@@ -36,6 +37,14 @@ interface GetNextTokenOptions {
type TokenEntry = [string, TokenData];
interface CircuitBreakerEntry {
failureCount: number;
lastFailureTime: number | null;
state: 'healthy' | 'degraded' | 'unhealthy';
failureRate: number;
requestLog: Array<{ timestamp: number; success: boolean }>;
}
function normalizeProjectId(projectId: string | null | undefined): string | undefined {
if (!isString(projectId)) {
return undefined;
@@ -89,6 +98,11 @@ export class TokenManagerService implements OnModuleInit {
private accountCooldowns: Map<string, number> = new Map();
private sessionBindings: Map<string, { accountId: string; expiresAt: number }> = new Map();
private rateLimitTracker = new RateLimitTracker();
private circuitBreakerStates = new Map<string, CircuitBreakerEntry>();
private readonly circuitBreakerWindowMs = 5 * 60 * 1000;
private readonly circuitBreakerUnhealthyThreshold = 5;
private readonly circuitBreakerDegradedThreshold = 2;
private readonly circuitBreakerFailureRateThreshold = 0.5;
private shadowComparisonCount = 0;
private shadowMismatchCount = 0;
@@ -96,6 +110,10 @@ export class TokenManagerService implements OnModuleInit {
private parityErrorCount = 0;
private noGoBlocked = false;
constructor() {
registerProxyAdvancedService('tokenManager', this);
}
async onModuleInit() {
await this.loadAccounts();
}
@@ -115,6 +133,13 @@ export class TokenManagerService implements OnModuleInit {
}
}
// Prune circuit breaker states for removed accounts
for (const accountId of this.circuitBreakerStates.keys()) {
if (!this.tokens.has(accountId)) {
this.circuitBreakerStates.delete(accountId);
}
}
this.logger.log(`Token manager loaded ${count} cloud accounts into cache`);
return count;
} catch (e) {
@@ -331,18 +356,24 @@ export class TokenManagerService implements OnModuleInit {
);
}
if (candidateAccountPool.length === 0) {
// Circuit breaker: skip unhealthy accounts unless all are unhealthy
const healthyPool = candidateAccountPool.filter(
([accountId]) => this.getOrCreateCircuitBreakerEntry(accountId).state !== 'unhealthy',
);
const finalCandidatePool = healthyPool.length > 0 ? healthyPool : candidateAccountPool;
if (finalCandidatePool.length === 0) {
this.logger.warn('No eligible account found after exclusion filtering');
return null;
}
if (this.shouldExecuteShadowComparison()) {
this.executeShadowComparison(candidateAccountPool, sessionKey, model);
this.executeShadowComparison(finalCandidatePool, sessionKey, model);
}
const selectedTokenEntry = this.isParitySchedulingEnabled()
? await this.selectParityTokenCandidate(candidateAccountPool, sessionKey, model, now)
: this.selectLegacyTokenCandidate(candidateAccountPool, sessionKey, now);
? await this.selectParityTokenCandidate(finalCandidatePool, sessionKey, model, now)
: this.selectLegacyTokenCandidate(finalCandidatePool, sessionKey, now);
if (!selectedTokenEntry) {
return null;
@@ -978,6 +1009,95 @@ export class TokenManagerService implements OnModuleInit {
return this.tokens.size;
}
recordSuccess(accountId: string): void {
const resolvedId = this.resolveAccountId(accountId) ?? accountId;
const entry = this.getOrCreateCircuitBreakerEntry(resolvedId);
entry.failureCount = 0;
entry.requestLog.push({ timestamp: Date.now(), success: true });
this.computeCircuitBreakerState(entry);
}
recordFailure(accountId: string, error?: string): void {
const resolvedId = this.resolveAccountId(accountId) ?? accountId;
const entry = this.getOrCreateCircuitBreakerEntry(resolvedId);
entry.failureCount++;
entry.lastFailureTime = Date.now();
entry.requestLog.push({ timestamp: Date.now(), success: false });
this.computeCircuitBreakerState(entry);
if (entry.state === 'unhealthy') {
this.logger.warn(
`Account ${resolvedId} circuit breaker opened: failureCount=${entry.failureCount}, failureRate=${entry.failureRate.toFixed(2)}${error ? `, error=${error}` : ''}`,
);
} else if (entry.state === 'degraded') {
this.logger.warn(
`Account ${resolvedId} circuit breaker degraded: failureCount=${entry.failureCount}, failureRate=${entry.failureRate.toFixed(2)}${error ? `, error=${error}` : ''}`,
);
}
}
getCircuitBreakerStatus(): Record<string, { state: 'healthy' | 'degraded' | 'unhealthy'; failureCount: number; lastFailureTime: number | null; failureRate: number }> {
const result: Record<string, { state: 'healthy' | 'degraded' | 'unhealthy'; failureCount: number; lastFailureTime: number | null; failureRate: number }> = {};
for (const accountId of this.tokens.keys()) {
const entry = this.getOrCreateCircuitBreakerEntry(accountId);
this.computeCircuitBreakerState(entry);
result[accountId] = {
state: entry.state,
failureCount: entry.failureCount,
lastFailureTime: entry.lastFailureTime,
failureRate: entry.failureRate,
};
}
return result;
}
getParityCounters(): { totalRequests: number; matchedRequests: number; parityViolations: number; lastUpdated: string } {
return {
totalRequests: this.parityRequestCount,
matchedRequests: this.shadowComparisonCount,
parityViolations: this.parityErrorCount,
lastUpdated: new Date().toISOString(),
};
}
private getOrCreateCircuitBreakerEntry(accountId: string): CircuitBreakerEntry {
let entry = this.circuitBreakerStates.get(accountId);
if (!entry) {
entry = {
failureCount: 0,
lastFailureTime: null,
state: 'healthy',
failureRate: 0,
requestLog: [],
};
this.circuitBreakerStates.set(accountId, entry);
}
return entry;
}
private pruneCircuitBreakerLog(entry: CircuitBreakerEntry): void {
const cutoff = Date.now() - this.circuitBreakerWindowMs;
entry.requestLog = entry.requestLog.filter((r) => r.timestamp > cutoff);
}
private computeCircuitBreakerState(entry: CircuitBreakerEntry): void {
this.pruneCircuitBreakerLog(entry);
const total = entry.requestLog.length;
const failures = entry.requestLog.filter((r) => !r.success).length;
entry.failureRate = total > 0 ? failures / total : 0;
if (entry.failureCount >= this.circuitBreakerUnhealthyThreshold) {
entry.state = 'unhealthy';
} else if (
entry.failureCount >= this.circuitBreakerDegradedThreshold &&
entry.failureRate >= this.circuitBreakerFailureRateThreshold
) {
entry.state = 'degraded';
} else {
entry.state = 'healthy';
}
}
private normalizeRefreshedOauthClientKey(
currentToken: { oauth_client_key?: string; project_id?: string },
refreshedClientKey?: string,