feat(proxy): enrich traffic logs with model and token metadata
- Add model to logInbound metadata - Add model, tokensPrompt, tokensCompletion, tokensTotal to logOutbound metadata - Enrich SSE streaming with token metadata from final usage chunk - Add metadata parameter to logOutbound method signature - Data persisted in traffic_logs.metadata_json for per-request cost calculation
This commit is contained in:
1 parent
99e458f69b
commit
641c739b64
2 files changed
+33
-5
No files matched your search
@@ -882,6 +882,7 @@ export class ProxyController {
|
|||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
trafficLogger.logInbound(requestId, '/v1/chat/completions', body, undefined, {
|
trafficLogger.logInbound(requestId, '/v1/chat/completions', body, undefined, {
|
||||||
stream: body.stream,
|
stream: body.stream,
|
||||||
|
model: body.model,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Extract x-session-affinity header so OpenCode session stickiness works even when
|
// Extract x-session-affinity header so OpenCode session stickiness works even when
|
||||||
@@ -908,12 +909,21 @@ export class ProxyController {
|
|||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
const duration = Date.now() - startTime;
|
const duration = Date.now() - startTime;
|
||||||
|
const response = result as OpenAIChatResponse;
|
||||||
|
const usage = response.usage;
|
||||||
trafficLogger.logOutbound(
|
trafficLogger.logOutbound(
|
||||||
requestId,
|
requestId,
|
||||||
'/v1/chat/completions',
|
'/v1/chat/completions',
|
||||||
HttpStatus.OK,
|
HttpStatus.OK,
|
||||||
result,
|
result,
|
||||||
duration,
|
duration,
|
||||||
|
undefined,
|
||||||
|
{
|
||||||
|
model: response.model || body.model,
|
||||||
|
tokensPrompt: usage?.prompt_tokens ?? 0,
|
||||||
|
tokensCompletion: usage?.completion_tokens ?? 0,
|
||||||
|
tokensTotal: usage?.total_tokens ?? 0,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
this.proxyMetrics?.recordRequest({
|
this.proxyMetrics?.recordRequest({
|
||||||
timestamp: startTime,
|
timestamp: startTime,
|
||||||
@@ -921,9 +931,8 @@ export class ProxyController {
|
|||||||
status: HttpStatus.OK,
|
status: HttpStatus.OK,
|
||||||
endpoint: '/v1/chat/completions',
|
endpoint: '/v1/chat/completions',
|
||||||
});
|
});
|
||||||
const response = result as OpenAIChatResponse;
|
if (usage?.total_tokens) {
|
||||||
if (response.usage?.total_tokens) {
|
this.proxyMetrics?.recordTokens(usage.total_tokens);
|
||||||
this.proxyMetrics?.recordTokens(response.usage.total_tokens);
|
|
||||||
}
|
}
|
||||||
this.setProxyHeaders(res, proxyContext);
|
this.setProxyHeaders(res, proxyContext);
|
||||||
res.status(HttpStatus.OK).send(result);
|
res.status(HttpStatus.OK).send(result);
|
||||||
@@ -1498,7 +1507,7 @@ export class ProxyController {
|
|||||||
return isObjectLike(value) && isFunction((value as { subscribe?: unknown }).subscribe);
|
return isObjectLike(value) && isFunction((value as { subscribe?: unknown }).subscribe);
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractAndRecordTokens(payload: string): void {
|
private extractAndRecordTokens(payload: string, requestId?: string, endpoint?: string): void {
|
||||||
try {
|
try {
|
||||||
const lines = payload.split('\n');
|
const lines = payload.split('\n');
|
||||||
for (const line of lines) {
|
for (const line of lines) {
|
||||||
@@ -1514,11 +1523,28 @@ export class ProxyController {
|
|||||||
const usageMetadata = parsed?.usageMetadata;
|
const usageMetadata = parsed?.usageMetadata;
|
||||||
if (usageMetadata && typeof usageMetadata.totalTokenCount === 'number') {
|
if (usageMetadata && typeof usageMetadata.totalTokenCount === 'number') {
|
||||||
this.proxyMetrics?.recordTokens(usageMetadata.totalTokenCount);
|
this.proxyMetrics?.recordTokens(usageMetadata.totalTokenCount);
|
||||||
|
// Log final token counts for this stream
|
||||||
|
if (requestId && endpoint) {
|
||||||
|
trafficLogger.logSseChunk(requestId, endpoint, undefined, {
|
||||||
|
model: parsed?.model,
|
||||||
|
tokensPrompt: usageMetadata.promptTokenCount ?? 0,
|
||||||
|
tokensCompletion: usageMetadata.candidatesTokenCount ?? 0,
|
||||||
|
tokensTotal: usageMetadata.totalTokenCount,
|
||||||
|
});
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const usage = parsed?.usage;
|
const usage = parsed?.usage;
|
||||||
if (usage && typeof usage.total_tokens === 'number') {
|
if (usage && typeof usage.total_tokens === 'number') {
|
||||||
this.proxyMetrics?.recordTokens(usage.total_tokens);
|
this.proxyMetrics?.recordTokens(usage.total_tokens);
|
||||||
|
if (requestId && endpoint) {
|
||||||
|
trafficLogger.logSseChunk(requestId, endpoint, undefined, {
|
||||||
|
model: parsed?.model,
|
||||||
|
tokensPrompt: usage.prompt_tokens ?? 0,
|
||||||
|
tokensCompletion: usage.completion_tokens ?? 0,
|
||||||
|
tokensTotal: usage.total_tokens,
|
||||||
|
});
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1606,7 +1632,7 @@ export class ProxyController {
|
|||||||
if (requestId && endpoint) {
|
if (requestId && endpoint) {
|
||||||
trafficLogger.logSseChunk(requestId, endpoint, chunk);
|
trafficLogger.logSseChunk(requestId, endpoint, chunk);
|
||||||
}
|
}
|
||||||
this.extractAndRecordTokens(payload);
|
this.extractAndRecordTokens(payload, requestId, endpoint);
|
||||||
if (isFunction((res.raw as any).flush)) {
|
if (isFunction((res.raw as any).flush)) {
|
||||||
(res.raw as any).flush();
|
(res.raw as any).flush();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ class TrafficLogger {
|
|||||||
body: unknown,
|
body: unknown,
|
||||||
durationMs: number,
|
durationMs: number,
|
||||||
headers?: Record<string, unknown>,
|
headers?: Record<string, unknown>,
|
||||||
|
metadata?: Record<string, unknown>,
|
||||||
): void {
|
): void {
|
||||||
this.log({
|
this.log({
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
@@ -177,6 +178,7 @@ class TrafficLogger {
|
|||||||
headers: this.sanitizeHeaders(headers),
|
headers: this.sanitizeHeaders(headers),
|
||||||
body: this.truncateBody(body),
|
body: this.truncateBody(body),
|
||||||
durationMs,
|
durationMs,
|
||||||
|
metadata,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user