diff --git a/docs/telemetry.md b/docs/telemetry.md index 9153e4d7..b308217f 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -154,6 +154,7 @@ Logs are timestamped records of specific events. The following events are logged - **Attributes**: - `prompt_length` - `prompt` (this attribute is excluded if `log_prompts_enabled` is configured to be `false`) + - `auth_type` - `gemini_cli.tool_call`: This event occurs for each function call. - **Attributes**: @@ -177,6 +178,7 @@ Logs are timestamped records of specific events. The following events are logged - `error_type` - `status_code` - `duration_ms` + - `auth_type` - `gemini_cli.api_response`: This event occurs upon receiving a response from Gemini API. - **Attributes**: @@ -190,6 +192,7 @@ Logs are timestamped records of specific events. The following events are logged - `thoughts_token_count` - `tool_token_count` - `response_text` (if applicable) + - `auth_type` ### Metrics diff --git a/packages/cli/src/gemini.tsx b/packages/cli/src/gemini.tsx index 8b58c46a..23990f6d 100644 --- a/packages/cli/src/gemini.tsx +++ b/packages/cli/src/gemini.tsx @@ -212,6 +212,7 @@ export async function main() { 'event.timestamp': new Date().toISOString(), prompt: input, prompt_id, + auth_type: config.getContentGeneratorConfig().authType!, prompt_length: input.length, }); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index e0e21f55..7e45cab2 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -273,6 +273,13 @@ describe('useGeminiStream', () => { return clientInstance; }); + const contentGeneratorConfig = { + model: 'test-model', + apiKey: 'test-key', + vertexai: false, + authType: AuthType.USE_GEMINI, + }; + mockConfig = { apiKey: 'test-api-key', model: 'gemini-pro', @@ -307,6 +314,9 @@ describe('useGeminiStream', () => { }, setQuotaErrorOccurred: vi.fn(), getQuotaErrorOccurred: vi.fn(() => false), + getContentGeneratorConfig: vi + .fn() + .mockReturnValue(contentGeneratorConfig), } as unknown as Config; mockOnDebugMessage = vi.fn(); mockHandleSlashCommand = vi.fn().mockResolvedValue(false); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index a9326528..05fb9835 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -223,7 +223,12 @@ export const useGeminiStream = ( const trimmedQuery = query.trim(); logUserPrompt( config, - new UserPromptEvent(trimmedQuery.length, prompt_id, trimmedQuery), + new UserPromptEvent( + trimmedQuery.length, + prompt_id, + config.getContentGeneratorConfig().authType!, + trimmedQuery, + ), ); onDebugMessage(`User query: '${trimmedQuery}'`); await logger?.logMessage(MessageSenderType.USER, trimmedQuery); diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index f57425a3..ab4f602c 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -172,6 +172,7 @@ export class GeminiChat { this.config.getModel(), durationMs, prompt_id, + this.config.getContentGeneratorConfig().authType!, usageMetadata, responseText, ), @@ -193,6 +194,7 @@ export class GeminiChat { errorMessage, durationMs, prompt_id, + this.config.getContentGeneratorConfig().authType!, errorType, ), ); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 673233f3..b75d87ef 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -266,6 +266,10 @@ export class ClearcutLogger { gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID, value: JSON.stringify(event.prompt_id), }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE, + value: JSON.stringify(event.auth_type), + }, ]; this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data)); @@ -377,6 +381,10 @@ export class ClearcutLogger { EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT, value: JSON.stringify(event.tool_token_count), }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE, + value: JSON.stringify(event.auth_type), + }, ]; this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data)); @@ -407,6 +415,10 @@ export class ClearcutLogger { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS, value: JSON.stringify(event.duration_ms), }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE, + value: JSON.stringify(event.auth_type), + }, ]; this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data)); diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index e8a74936..577c5c07 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -144,6 +144,9 @@ export enum EventMetadataKey { // Logs the Prompt Id GEMINI_CLI_PROMPT_ID = 35, + + // Logs the Auth type for the prompt, api responses and errors. + GEMINI_CLI_AUTH_TYPE = 36, } export function getEventMetadataKey( diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index e7dd721f..13617946 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -127,7 +127,12 @@ describe('loggers', () => { } as unknown as Config; it('should log a user prompt', () => { - const event = new UserPromptEvent(11, 'prompt-id-8', 'test-prompt'); + const event = new UserPromptEvent( + 11, + 'prompt-id-8', + AuthType.USE_VERTEX_AI, + 'test-prompt', + ); logUserPrompt(mockConfig, event); @@ -151,7 +156,11 @@ describe('loggers', () => { getTargetDir: () => 'target-dir', getUsageStatisticsEnabled: () => true, } as unknown as Config; - const event = new UserPromptEvent(11, 'test-prompt'); + const event = new UserPromptEvent( + 11, + 'test-prompt', + AuthType.CLOUD_SHELL, + ); logUserPrompt(mockConfig, event); @@ -202,6 +211,7 @@ describe('loggers', () => { 'test-model', 100, 'prompt-id-1', + AuthType.LOGIN_WITH_GOOGLE, usageData, 'test-response', ); @@ -226,6 +236,7 @@ describe('loggers', () => { total_token_count: 0, response_text: 'test-response', prompt_id: 'prompt-id-1', + auth_type: 'oauth-personal', }, }); @@ -263,6 +274,7 @@ describe('loggers', () => { 'test-model', 100, 'prompt-id-1', + AuthType.USE_GEMINI, usageData, 'test-response', 'test-error', diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 46f86b89..9f6e1462 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -96,13 +96,20 @@ export class UserPromptEvent { 'event.timestamp': string; // ISO 8601 prompt_length: number; prompt_id: string; + auth_type: string; prompt?: string; - constructor(prompt_length: number, prompt_Id: string, prompt?: string) { + constructor( + prompt_length: number, + prompt_Id: string, + auth_type: string, + prompt?: string, + ) { this['event.name'] = 'user_prompt'; this['event.timestamp'] = new Date().toISOString(); this.prompt_length = prompt_length; this.prompt_id = prompt_Id; + this.auth_type = auth_type; this.prompt = prompt; } } @@ -160,12 +167,14 @@ export class ApiErrorEvent { status_code?: number | string; duration_ms: number; prompt_id: string; + auth_type: string; constructor( model: string, error: string, duration_ms: number, prompt_id: string, + auth_type: string, error_type?: string, status_code?: number | string, ) { @@ -177,6 +186,7 @@ export class ApiErrorEvent { this.status_code = status_code; this.duration_ms = duration_ms; this.prompt_id = prompt_id; + this.auth_type = auth_type; } } @@ -195,11 +205,13 @@ export class ApiResponseEvent { total_token_count: number; response_text?: string; prompt_id: string; + auth_type: string; constructor( model: string, duration_ms: number, prompt_id: string, + auth_type: string, usage_data?: GenerateContentResponseUsageMetadata, response_text?: string, error?: string, @@ -218,6 +230,7 @@ export class ApiResponseEvent { this.response_text = response_text; this.error = error; this.prompt_id = prompt_id; + this.auth_type = auth_type; } }