Adds Flash Fallback logging and clearcut logging (#3843)
This commit is contained in:
parent
764809753a
commit
5b5f496436
|
@ -194,6 +194,10 @@ Logs are timestamped records of specific events. The following events are logged
|
|||
- `response_text` (if applicable)
|
||||
- `auth_type`
|
||||
|
||||
- `gemini_cli.flash_fallback`: This event occurs when Gemini CLI switches to flash as fallback.
|
||||
- **Attributes**:
|
||||
- `auth_type`
|
||||
|
||||
### Metrics
|
||||
|
||||
Metrics are numerical measurements of behavior over time. The following metrics are collected for Gemini CLI:
|
||||
|
|
|
@ -54,6 +54,8 @@ import {
|
|||
ApprovalMode,
|
||||
isEditorAvailable,
|
||||
EditorType,
|
||||
FlashFallbackEvent,
|
||||
logFlashFallback,
|
||||
} from '@google/gemini-cli-core';
|
||||
import { validateAuthMethod } from '../config/auth.js';
|
||||
import { useLogger } from './hooks/useLogger.js';
|
||||
|
@ -340,6 +342,10 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
|
|||
config.setQuotaErrorOccurred(true);
|
||||
// Switch model for future use but return false to stop current retry
|
||||
config.setModel(fallbackModel);
|
||||
logFlashFallback(
|
||||
config,
|
||||
new FlashFallbackEvent(config.getContentGeneratorConfig().authType!),
|
||||
);
|
||||
return false; // Don't continue with current prompt
|
||||
};
|
||||
|
||||
|
|
|
@ -14,6 +14,7 @@ import {
|
|||
ApiRequestEvent,
|
||||
ApiResponseEvent,
|
||||
ApiErrorEvent,
|
||||
FlashFallbackEvent,
|
||||
} from '../types.js';
|
||||
import { EventMetadataKey } from './event-metadata-key.js';
|
||||
import { Config } from '../../config/config.js';
|
||||
|
@ -30,6 +31,7 @@ const api_request_event_name = 'api_request';
|
|||
const api_response_event_name = 'api_response';
|
||||
const api_error_event_name = 'api_error';
|
||||
const end_session_event_name = 'end_session';
|
||||
const flash_fallback_event_name = 'flash_fallback';
|
||||
|
||||
export interface LogResponse {
|
||||
nextRequestWaitMs?: number;
|
||||
|
@ -431,6 +433,20 @@ export class ClearcutLogger {
|
|||
this.flushIfNeeded();
|
||||
}
|
||||
|
||||
logFlashFallbackEvent(event: FlashFallbackEvent): void {
|
||||
const data = [
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
|
||||
value: JSON.stringify(event.auth_type),
|
||||
},
|
||||
];
|
||||
|
||||
this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
|
||||
this.flushToClearcut().catch((error) => {
|
||||
console.debug('Error flushing to Clearcut:', error);
|
||||
});
|
||||
}
|
||||
|
||||
logEndSessionEvent(event: EndSessionEvent): void {
|
||||
const data = [
|
||||
{
|
||||
|
|
|
@ -12,6 +12,7 @@ export const EVENT_API_REQUEST = 'gemini_cli.api_request';
|
|||
export const EVENT_API_ERROR = 'gemini_cli.api_error';
|
||||
export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
|
||||
export const EVENT_CLI_CONFIG = 'gemini_cli.config';
|
||||
export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback';
|
||||
|
||||
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
|
||||
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
|
||||
|
|
|
@ -25,6 +25,7 @@ export {
|
|||
logApiRequest,
|
||||
logApiError,
|
||||
logApiResponse,
|
||||
logFlashFallback,
|
||||
} from './loggers.js';
|
||||
export {
|
||||
StartSessionEvent,
|
||||
|
@ -35,6 +36,7 @@ export {
|
|||
ApiErrorEvent,
|
||||
ApiResponseEvent,
|
||||
TelemetryEvent,
|
||||
FlashFallbackEvent,
|
||||
} from './types.js';
|
||||
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
||||
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
||||
|
|
|
@ -23,6 +23,7 @@ import {
|
|||
EVENT_CLI_CONFIG,
|
||||
EVENT_TOOL_CALL,
|
||||
EVENT_USER_PROMPT,
|
||||
EVENT_FLASH_FALLBACK,
|
||||
} from './constants.js';
|
||||
import {
|
||||
logApiRequest,
|
||||
|
@ -30,6 +31,7 @@ import {
|
|||
logCliConfiguration,
|
||||
logUserPrompt,
|
||||
logToolCall,
|
||||
logFlashFallback,
|
||||
} from './loggers.js';
|
||||
import {
|
||||
ApiRequestEvent,
|
||||
|
@ -38,6 +40,7 @@ import {
|
|||
ToolCallDecision,
|
||||
ToolCallEvent,
|
||||
UserPromptEvent,
|
||||
FlashFallbackEvent,
|
||||
} from './types.js';
|
||||
import * as metrics from './metrics.js';
|
||||
import * as sdk from './sdk.js';
|
||||
|
@ -350,6 +353,29 @@ describe('loggers', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('logFlashFallback', () => {
|
||||
const mockConfig = {
|
||||
getSessionId: () => 'test-session-id',
|
||||
getUsageStatisticsEnabled: () => true,
|
||||
} as unknown as Config;
|
||||
|
||||
it('should log flash fallback event', () => {
|
||||
const event = new FlashFallbackEvent(AuthType.USE_VERTEX_AI);
|
||||
|
||||
logFlashFallback(mockConfig, event);
|
||||
|
||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||
body: 'Switching to flash as Fallback.',
|
||||
attributes: {
|
||||
'session.id': 'test-session-id',
|
||||
'event.name': EVENT_FLASH_FALLBACK,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
auth_type: 'vertex-ai',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('logToolCall', () => {
|
||||
const cfg1 = {
|
||||
getSessionId: () => 'test-session-id',
|
||||
|
|
|
@ -14,6 +14,7 @@ import {
|
|||
EVENT_CLI_CONFIG,
|
||||
EVENT_TOOL_CALL,
|
||||
EVENT_USER_PROMPT,
|
||||
EVENT_FLASH_FALLBACK,
|
||||
SERVICE_NAME,
|
||||
} from './constants.js';
|
||||
import {
|
||||
|
@ -23,6 +24,7 @@ import {
|
|||
StartSessionEvent,
|
||||
ToolCallEvent,
|
||||
UserPromptEvent,
|
||||
FlashFallbackEvent,
|
||||
} from './types.js';
|
||||
import {
|
||||
recordApiErrorMetrics,
|
||||
|
@ -156,6 +158,28 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void {
|
|||
logger.emit(logRecord);
|
||||
}
|
||||
|
||||
export function logFlashFallback(
|
||||
config: Config,
|
||||
event: FlashFallbackEvent,
|
||||
): void {
|
||||
ClearcutLogger.getInstance(config)?.logFlashFallbackEvent(event);
|
||||
if (!isTelemetrySdkInitialized()) return;
|
||||
|
||||
const attributes: LogAttributes = {
|
||||
...getCommonAttributes(config),
|
||||
...event,
|
||||
'event.name': EVENT_FLASH_FALLBACK,
|
||||
'event.timestamp': new Date().toISOString(),
|
||||
};
|
||||
|
||||
const logger = logs.getLogger(SERVICE_NAME);
|
||||
const logRecord: LogRecord = {
|
||||
body: `Switching to flash as Fallback.`,
|
||||
attributes,
|
||||
};
|
||||
logger.emit(logRecord);
|
||||
}
|
||||
|
||||
export function logApiError(config: Config, event: ApiErrorEvent): void {
|
||||
const uiEvent = {
|
||||
...event,
|
||||
|
|
|
@ -234,6 +234,18 @@ export class ApiResponseEvent {
|
|||
}
|
||||
}
|
||||
|
||||
export class FlashFallbackEvent {
|
||||
'event.name': 'flash_fallback';
|
||||
'event.timestamp': string; // ISO 8601
|
||||
auth_type: string;
|
||||
|
||||
constructor(auth_type: string) {
|
||||
this['event.name'] = 'flash_fallback';
|
||||
this['event.timestamp'] = new Date().toISOString();
|
||||
this.auth_type = auth_type;
|
||||
}
|
||||
}
|
||||
|
||||
export type TelemetryEvent =
|
||||
| StartSessionEvent
|
||||
| EndSessionEvent
|
||||
|
@ -241,4 +253,5 @@ export type TelemetryEvent =
|
|||
| ToolCallEvent
|
||||
| ApiRequestEvent
|
||||
| ApiErrorEvent
|
||||
| ApiResponseEvent;
|
||||
| ApiResponseEvent
|
||||
| FlashFallbackEvent;
|
||||
|
|
Loading…
Reference in New Issue