diff --git a/docs/telemetry.md b/docs/telemetry.md index b308217f..76958794 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -194,6 +194,10 @@ Logs are timestamped records of specific events. The following events are logged - `response_text` (if applicable) - `auth_type` +- `gemini_cli.flash_fallback`: This event occurs when Gemini CLI switches to flash as fallback. + - **Attributes**: + - `auth_type` + ### Metrics Metrics are numerical measurements of behavior over time. The following metrics are collected for Gemini CLI: diff --git a/packages/cli/src/ui/App.tsx b/packages/cli/src/ui/App.tsx index 519a4b53..f2324b28 100644 --- a/packages/cli/src/ui/App.tsx +++ b/packages/cli/src/ui/App.tsx @@ -54,6 +54,8 @@ import { ApprovalMode, isEditorAvailable, EditorType, + FlashFallbackEvent, + logFlashFallback, } from '@google/gemini-cli-core'; import { validateAuthMethod } from '../config/auth.js'; import { useLogger } from './hooks/useLogger.js'; @@ -340,6 +342,10 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => { config.setQuotaErrorOccurred(true); // Switch model for future use but return false to stop current retry config.setModel(fallbackModel); + logFlashFallback( + config, + new FlashFallbackEvent(config.getContentGeneratorConfig().authType!), + ); return false; // Don't continue with current prompt }; diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 42b1f6fc..cd2abe81 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -14,6 +14,7 @@ import { ApiRequestEvent, ApiResponseEvent, ApiErrorEvent, + FlashFallbackEvent, } from '../types.js'; import { EventMetadataKey } from './event-metadata-key.js'; import { Config } from '../../config/config.js'; @@ -30,6 +31,7 @@ const api_request_event_name = 'api_request'; const api_response_event_name = 'api_response'; const api_error_event_name = 'api_error'; const end_session_event_name = 'end_session'; +const flash_fallback_event_name = 'flash_fallback'; export interface LogResponse { nextRequestWaitMs?: number; @@ -431,6 +433,20 @@ export class ClearcutLogger { this.flushIfNeeded(); } + logFlashFallbackEvent(event: FlashFallbackEvent): void { + const data = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE, + value: JSON.stringify(event.auth_type), + }, + ]; + + this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data)); + this.flushToClearcut().catch((error) => { + console.debug('Error flushing to Clearcut:', error); + }); + } + logEndSessionEvent(event: EndSessionEvent): void { const data = [ { diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts index de760205..62c4bf24 100644 --- a/packages/core/src/telemetry/constants.ts +++ b/packages/core/src/telemetry/constants.ts @@ -12,6 +12,7 @@ export const EVENT_API_REQUEST = 'gemini_cli.api_request'; export const EVENT_API_ERROR = 'gemini_cli.api_error'; export const EVENT_API_RESPONSE = 'gemini_cli.api_response'; export const EVENT_CLI_CONFIG = 'gemini_cli.config'; +export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback'; export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count'; export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency'; diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index a17c8af3..8da31727 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -25,6 +25,7 @@ export { logApiRequest, logApiError, logApiResponse, + logFlashFallback, } from './loggers.js'; export { StartSessionEvent, @@ -35,6 +36,7 @@ export { ApiErrorEvent, ApiResponseEvent, TelemetryEvent, + FlashFallbackEvent, } from './types.js'; export { SpanStatusCode, ValueType } from '@opentelemetry/api'; export { SemanticAttributes } from '@opentelemetry/semantic-conventions'; diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 13617946..7a24bcca 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -23,6 +23,7 @@ import { EVENT_CLI_CONFIG, EVENT_TOOL_CALL, EVENT_USER_PROMPT, + EVENT_FLASH_FALLBACK, } from './constants.js'; import { logApiRequest, @@ -30,6 +31,7 @@ import { logCliConfiguration, logUserPrompt, logToolCall, + logFlashFallback, } from './loggers.js'; import { ApiRequestEvent, @@ -38,6 +40,7 @@ import { ToolCallDecision, ToolCallEvent, UserPromptEvent, + FlashFallbackEvent, } from './types.js'; import * as metrics from './metrics.js'; import * as sdk from './sdk.js'; @@ -350,6 +353,29 @@ describe('loggers', () => { }); }); + describe('logFlashFallback', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + getUsageStatisticsEnabled: () => true, + } as unknown as Config; + + it('should log flash fallback event', () => { + const event = new FlashFallbackEvent(AuthType.USE_VERTEX_AI); + + logFlashFallback(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Switching to flash as Fallback.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': EVENT_FLASH_FALLBACK, + 'event.timestamp': '2025-01-01T00:00:00.000Z', + auth_type: 'vertex-ai', + }, + }); + }); + }); + describe('logToolCall', () => { const cfg1 = { getSessionId: () => 'test-session-id', diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index a7231e2f..5929ec58 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -14,6 +14,7 @@ import { EVENT_CLI_CONFIG, EVENT_TOOL_CALL, EVENT_USER_PROMPT, + EVENT_FLASH_FALLBACK, SERVICE_NAME, } from './constants.js'; import { @@ -23,6 +24,7 @@ import { StartSessionEvent, ToolCallEvent, UserPromptEvent, + FlashFallbackEvent, } from './types.js'; import { recordApiErrorMetrics, @@ -156,6 +158,28 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void { logger.emit(logRecord); } +export function logFlashFallback( + config: Config, + event: FlashFallbackEvent, +): void { + ClearcutLogger.getInstance(config)?.logFlashFallbackEvent(event); + if (!isTelemetrySdkInitialized()) return; + + const attributes: LogAttributes = { + ...getCommonAttributes(config), + ...event, + 'event.name': EVENT_FLASH_FALLBACK, + 'event.timestamp': new Date().toISOString(), + }; + + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `Switching to flash as Fallback.`, + attributes, + }; + logger.emit(logRecord); +} + export function logApiError(config: Config, event: ApiErrorEvent): void { const uiEvent = { ...event, diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 9ac12050..d04756e2 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -234,6 +234,18 @@ export class ApiResponseEvent { } } +export class FlashFallbackEvent { + 'event.name': 'flash_fallback'; + 'event.timestamp': string; // ISO 8601 + auth_type: string; + + constructor(auth_type: string) { + this['event.name'] = 'flash_fallback'; + this['event.timestamp'] = new Date().toISOString(); + this.auth_type = auth_type; + } +} + export type TelemetryEvent = | StartSessionEvent | EndSessionEvent @@ -241,4 +253,5 @@ export type TelemetryEvent = | ToolCallEvent | ApiRequestEvent | ApiErrorEvent - | ApiResponseEvent; + | ApiResponseEvent + | FlashFallbackEvent;