Adds Flash Fallback logging and clearcut logging (#3843)

This commit is contained in:
uttamkanodia14 2025-07-12 02:40:25 +05:30 committed by GitHub
parent 764809753a
commit 5b5f496436
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 93 additions and 1 deletions

View File

@ -194,6 +194,10 @@ Logs are timestamped records of specific events. The following events are logged
- `response_text` (if applicable)
- `auth_type`
- `gemini_cli.flash_fallback`: This event occurs when Gemini CLI switches to flash as fallback.
- **Attributes**:
- `auth_type`
### Metrics
Metrics are numerical measurements of behavior over time. The following metrics are collected for Gemini CLI:

View File

@ -54,6 +54,8 @@ import {
ApprovalMode,
isEditorAvailable,
EditorType,
FlashFallbackEvent,
logFlashFallback,
} from '@google/gemini-cli-core';
import { validateAuthMethod } from '../config/auth.js';
import { useLogger } from './hooks/useLogger.js';
@ -340,6 +342,10 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
config.setQuotaErrorOccurred(true);
// Switch model for future use but return false to stop current retry
config.setModel(fallbackModel);
logFlashFallback(
config,
new FlashFallbackEvent(config.getContentGeneratorConfig().authType!),
);
return false; // Don't continue with current prompt
};

View File

@ -14,6 +14,7 @@ import {
ApiRequestEvent,
ApiResponseEvent,
ApiErrorEvent,
FlashFallbackEvent,
} from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js';
import { Config } from '../../config/config.js';
@ -30,6 +31,7 @@ const api_request_event_name = 'api_request';
const api_response_event_name = 'api_response';
const api_error_event_name = 'api_error';
const end_session_event_name = 'end_session';
const flash_fallback_event_name = 'flash_fallback';
export interface LogResponse {
nextRequestWaitMs?: number;
@ -431,6 +433,20 @@ export class ClearcutLogger {
this.flushIfNeeded();
}
logFlashFallbackEvent(event: FlashFallbackEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
logEndSessionEvent(event: EndSessionEvent): void {
const data = [
{

View File

@ -12,6 +12,7 @@ export const EVENT_API_REQUEST = 'gemini_cli.api_request';
export const EVENT_API_ERROR = 'gemini_cli.api_error';
export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
export const EVENT_CLI_CONFIG = 'gemini_cli.config';
export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback';
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';

View File

@ -25,6 +25,7 @@ export {
logApiRequest,
logApiError,
logApiResponse,
logFlashFallback,
} from './loggers.js';
export {
StartSessionEvent,
@ -35,6 +36,7 @@ export {
ApiErrorEvent,
ApiResponseEvent,
TelemetryEvent,
FlashFallbackEvent,
} from './types.js';
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';

View File

@ -23,6 +23,7 @@ import {
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
EVENT_FLASH_FALLBACK,
} from './constants.js';
import {
logApiRequest,
@ -30,6 +31,7 @@ import {
logCliConfiguration,
logUserPrompt,
logToolCall,
logFlashFallback,
} from './loggers.js';
import {
ApiRequestEvent,
@ -38,6 +40,7 @@ import {
ToolCallDecision,
ToolCallEvent,
UserPromptEvent,
FlashFallbackEvent,
} from './types.js';
import * as metrics from './metrics.js';
import * as sdk from './sdk.js';
@ -350,6 +353,29 @@ describe('loggers', () => {
});
});
describe('logFlashFallback', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
it('should log flash fallback event', () => {
const event = new FlashFallbackEvent(AuthType.USE_VERTEX_AI);
logFlashFallback(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Switching to flash as Fallback.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_FLASH_FALLBACK,
'event.timestamp': '2025-01-01T00:00:00.000Z',
auth_type: 'vertex-ai',
},
});
});
});
describe('logToolCall', () => {
const cfg1 = {
getSessionId: () => 'test-session-id',

View File

@ -14,6 +14,7 @@ import {
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
EVENT_FLASH_FALLBACK,
SERVICE_NAME,
} from './constants.js';
import {
@ -23,6 +24,7 @@ import {
StartSessionEvent,
ToolCallEvent,
UserPromptEvent,
FlashFallbackEvent,
} from './types.js';
import {
recordApiErrorMetrics,
@ -156,6 +158,28 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void {
logger.emit(logRecord);
}
export function logFlashFallback(
config: Config,
event: FlashFallbackEvent,
): void {
ClearcutLogger.getInstance(config)?.logFlashFallbackEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_FLASH_FALLBACK,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Switching to flash as Fallback.`,
attributes,
};
logger.emit(logRecord);
}
export function logApiError(config: Config, event: ApiErrorEvent): void {
const uiEvent = {
...event,

View File

@ -234,6 +234,18 @@ export class ApiResponseEvent {
}
}
export class FlashFallbackEvent {
'event.name': 'flash_fallback';
'event.timestamp': string; // ISO 8601
auth_type: string;
constructor(auth_type: string) {
this['event.name'] = 'flash_fallback';
this['event.timestamp'] = new Date().toISOString();
this.auth_type = auth_type;
}
}
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
@ -241,4 +253,5 @@ export type TelemetryEvent =
| ToolCallEvent
| ApiRequestEvent
| ApiErrorEvent
| ApiResponseEvent;
| ApiResponseEvent
| FlashFallbackEvent;