diff --git a/packages/cli/src/config/config.test.ts b/packages/cli/src/config/config.test.ts index 5b24f434..cc1310dd 100644 --- a/packages/cli/src/config/config.test.ts +++ b/packages/cli/src/config/config.test.ts @@ -11,13 +11,11 @@ import { Settings } from './settings.js'; import { Extension } from './extension.js'; import * as ServerConfig from '@gemini-cli/core'; -const MOCK_HOME_DIR = '/mock/home/user'; - vi.mock('os', async (importOriginal) => { const actualOs = await importOriginal(); return { ...actualOs, - homedir: vi.fn(() => MOCK_HOME_DIR), + homedir: vi.fn(() => '/mock/home/user'), }; }); @@ -53,7 +51,7 @@ describe('loadCliConfig', () => { beforeEach(() => { vi.resetAllMocks(); - vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR); + vi.mocked(os.homedir).mockReturnValue('/mock/home/user'); process.env.GEMINI_API_KEY = 'test-api-key'; // Ensure API key is set for tests }); @@ -98,7 +96,7 @@ describe('loadCliConfig telemetry', () => { beforeEach(() => { vi.resetAllMocks(); - vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR); + vi.mocked(os.homedir).mockReturnValue('/mock/home/user'); process.env.GEMINI_API_KEY = 'test-api-key'; }); @@ -250,7 +248,7 @@ describe('loadCliConfig telemetry', () => { describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => { beforeEach(() => { vi.resetAllMocks(); - vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR); + vi.mocked(os.homedir).mockReturnValue('/mock/home/user'); // Other common mocks would be reset here. }); @@ -310,7 +308,7 @@ describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => { // Example of a previously failing test structure: /* it('should correctly use mocked homedir for global path', async () => { - const MOCK_GEMINI_DIR_LOCAL = path.join(MOCK_HOME_DIR, '.gemini'); + const MOCK_GEMINI_DIR_LOCAL = path.join('/mock/home/user', '.gemini'); const MOCK_GLOBAL_PATH_LOCAL = path.join(MOCK_GEMINI_DIR_LOCAL, 'GEMINI.md'); mockFs({ [MOCK_GLOBAL_PATH_LOCAL]: { type: 'file', content: 'GlobalContentOnly' } diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 26894bc9..d8502bdd 100644 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -226,6 +226,7 @@ export async function loadCliConfig( process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? settings.telemetry?.otlpEndpoint, logPrompts: argv.telemetryLogPrompts ?? settings.telemetry?.logPrompts, + disableDataCollection: settings.telemetry?.disableDataCollection ?? true, }, // Git-aware file filtering settings fileFiltering: { diff --git a/packages/cli/src/config/settings.test.ts b/packages/cli/src/config/settings.test.ts index cfa5e2b5..efaddab3 100644 --- a/packages/cli/src/config/settings.test.ts +++ b/packages/cli/src/config/settings.test.ts @@ -6,15 +6,13 @@ /// -const MOCK_HOME_DIR = '/mock/home/user'; // MUST BE FIRST - -// Mock 'os' first. Its factory uses MOCK_HOME_DIR. +// Mock 'os' first. import * as osActual from 'os'; // Import for type info for the mock factory vi.mock('os', async (importOriginal) => { const actualOs = await importOriginal(); return { ...actualOs, - homedir: vi.fn(() => MOCK_HOME_DIR), + homedir: vi.fn(() => '/mock/home/user'), }; }); @@ -77,7 +75,7 @@ describe('Settings Loading and Merging', () => { mockFsMkdirSync = vi.mocked(fs.mkdirSync); mockStripJsonComments = vi.mocked(stripJsonComments); - vi.mocked(osActual.homedir).mockReturnValue(MOCK_HOME_DIR); + vi.mocked(osActual.homedir).mockReturnValue('/mock/home/user'); (mockStripJsonComments as unknown as Mock).mockImplementation( (jsonString: string) => jsonString, ); diff --git a/packages/cli/src/gemini.tsx b/packages/cli/src/gemini.tsx index 9657ba8f..293d50a5 100644 --- a/packages/cli/src/gemini.tsx +++ b/packages/cli/src/gemini.tsx @@ -131,6 +131,8 @@ export async function main() { } logUserPrompt(config, { + 'event.name': 'user_prompt', + 'event.timestamp': new Date().toISOString(), prompt: input, prompt_length: input.length, }); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index f8cc61bc..cb5b35b4 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -38,12 +38,17 @@ const MockedGeminiClientClass = vi.hoisted(() => }), ); +const MockedUserPromptEvent = vi.hoisted(() => + vi.fn().mockImplementation(() => {}), +); + vi.mock('@gemini-cli/core', async (importOriginal) => { const actualCoreModule = (await importOriginal()) as any; return { ...actualCoreModule, GitService: vi.fn(), GeminiClient: MockedGeminiClientClass, + UserPromptEvent: MockedUserPromptEvent, }; }); @@ -283,6 +288,7 @@ describe('useGeminiStream', () => { getProjectRoot: vi.fn(() => '/test/dir'), getCheckpointingEnabled: vi.fn(() => false), getGeminiClient: mockGetGeminiClient, + getDisableDataCollection: () => false, addHistory: vi.fn(), } as unknown as Config; mockOnDebugMessage = vi.fn(); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 09b14666..921fbdb1 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -23,6 +23,7 @@ import { EditorType, ThoughtSummary, isAuthError, + UserPromptEvent, } from '@gemini-cli/core'; import { type Part, type PartListUnion } from '@google/genai'; import { @@ -213,10 +214,10 @@ export const useGeminiStream = ( if (typeof query === 'string') { const trimmedQuery = query.trim(); - logUserPrompt(config, { - prompt: trimmedQuery, - prompt_length: trimmedQuery.length, - }); + logUserPrompt( + config, + new UserPromptEvent(trimmedQuery.length, trimmedQuery), + ); onDebugMessage(`User query: '${trimmedQuery}'`); await logger?.logMessage(MessageSenderType.USER, trimmedQuery); diff --git a/packages/cli/src/ui/hooks/useToolScheduler.test.ts b/packages/cli/src/ui/hooks/useToolScheduler.test.ts index 8e3f139b..4c8901dc 100644 --- a/packages/cli/src/ui/hooks/useToolScheduler.test.ts +++ b/packages/cli/src/ui/hooks/useToolScheduler.test.ts @@ -48,6 +48,7 @@ const mockToolRegistry = { const mockConfig = { getToolRegistry: vi.fn(() => mockToolRegistry as unknown as ToolRegistry), getApprovalMode: vi.fn(() => ApprovalMode.DEFAULT), + getDisableDataCollection: () => false, }; const mockTool: Tool = { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index be21ac8c..f6128e11 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -33,8 +33,10 @@ import { DEFAULT_TELEMETRY_TARGET, DEFAULT_OTLP_ENDPOINT, TelemetryTarget, + StartSessionEvent, } from '../telemetry/index.js'; import { DEFAULT_GEMINI_EMBEDDING_MODEL } from './models.js'; +import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js'; export enum ApprovalMode { DEFAULT = 'default', @@ -55,6 +57,7 @@ export interface TelemetrySettings { target?: TelemetryTarget; otlpEndpoint?: string; logPrompts?: boolean; + disableDataCollection?: boolean; } export class MCPServerConfig { @@ -114,6 +117,7 @@ export interface ConfigParameters { fileDiscoveryService?: FileDiscoveryService; bugCommand?: BugCommandSettings; model: string; + disableDataCollection?: boolean; } export class Config { @@ -150,6 +154,7 @@ export class Config { private readonly cwd: string; private readonly bugCommand: BugCommandSettings | undefined; private readonly model: string; + private readonly disableDataCollection: boolean; constructor(params: ConfigParameters) { this.sessionId = params.sessionId; @@ -189,6 +194,8 @@ export class Config { this.fileDiscoveryService = params.fileDiscoveryService ?? null; this.bugCommand = params.bugCommand; this.model = params.model; + this.disableDataCollection = + params.telemetry?.disableDataCollection ?? true; if (params.contextFileName) { setGeminiMdFilename(params.contextFileName); @@ -197,6 +204,12 @@ export class Config { if (this.telemetrySettings.enabled) { initializeTelemetry(this); } + + if (!this.disableDataCollection) { + ClearcutLogger.getInstance(this)?.enqueueLogEvent( + new StartSessionEvent(this), + ); + } } async refreshAuth(authMethod: AuthType) { @@ -370,6 +383,10 @@ export class Config { return this.fileDiscoveryService; } + getDisableDataCollection(): boolean { + return this.disableDataCollection; + } + async getGitService(): Promise { if (!this.gitService) { this.gitService = new GitService(this.targetDir); diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index 63feb874..656f8952 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -77,6 +77,7 @@ describe('CoreToolScheduler', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getDisableDataCollection: () => false, } as Config; const scheduler = new CoreToolScheduler({ diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index 14a39792..5e41f0c5 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -16,6 +16,7 @@ import { EditorType, Config, logToolCall, + ToolCallEvent, } from '../index.js'; import { Part, PartListUnion } from '@google/genai'; import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js'; @@ -652,20 +653,7 @@ export class CoreToolScheduler { this.toolCalls = []; for (const call of completedCalls) { - logToolCall( - this.config, - { - function_name: call.request.name, - function_args: call.request.args, - duration_ms: call.durationMs ?? 0, - success: call.status === 'success', - error: - call.status === 'error' - ? call.response.error?.message - : undefined, - }, - call.outcome, - ); + logToolCall(this.config, new ToolCallEvent(call)); } if (this.onAllToolCallsComplete) { diff --git a/packages/core/src/core/geminiChat.test.ts b/packages/core/src/core/geminiChat.test.ts index 9961103d..45c9b06e 100644 --- a/packages/core/src/core/geminiChat.test.ts +++ b/packages/core/src/core/geminiChat.test.ts @@ -27,6 +27,7 @@ const mockModelsModule = { const mockConfig = { getSessionId: () => 'test-session-id', getTelemetryLogPromptsEnabled: () => true, + getDisableDataCollection: () => false, } as unknown as Config; describe('GeminiChat', () => { diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index 3929dd26..e08aaf86 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -24,12 +24,16 @@ import { logApiRequest, logApiResponse, logApiError, - getFinalUsageMetadata, } from '../telemetry/loggers.js'; import { getStructuredResponse, getStructuredResponseFromParts, } from '../utils/generateContentResponseUtilities.js'; +import { + ApiErrorEvent, + ApiRequestEvent, + ApiResponseEvent, +} from '../telemetry/types.js'; /** * Returns true if the response is valid, false otherwise. @@ -152,14 +156,8 @@ export class GeminiChat { contents: Content[], model: string, ): Promise { - const shouldLogUserPrompts = (config: Config): boolean => - config.getTelemetryLogPromptsEnabled() ?? false; - const requestText = this._getRequestTextFromContents(contents); - logApiRequest(this.config, { - model, - request_text: shouldLogUserPrompts(this.config) ? requestText : undefined, - }); + logApiRequest(this.config, new ApiRequestEvent(model, requestText)); } private async _logApiResponse( @@ -167,31 +165,20 @@ export class GeminiChat { usageMetadata?: GenerateContentResponseUsageMetadata, responseText?: string, ): Promise { - logApiResponse(this.config, { - model: this.model, - duration_ms: durationMs, - status_code: 200, // Assuming 200 for success - input_token_count: usageMetadata?.promptTokenCount ?? 0, - output_token_count: usageMetadata?.candidatesTokenCount ?? 0, - cached_content_token_count: usageMetadata?.cachedContentTokenCount ?? 0, - thoughts_token_count: usageMetadata?.thoughtsTokenCount ?? 0, - tool_token_count: usageMetadata?.toolUsePromptTokenCount ?? 0, - response_text: responseText, - }); + logApiResponse( + this.config, + new ApiResponseEvent(this.model, durationMs, usageMetadata, responseText), + ); } private _logApiError(durationMs: number, error: unknown): void { const errorMessage = error instanceof Error ? error.message : String(error); const errorType = error instanceof Error ? error.name : 'unknown'; - const statusCode = 'unknown'; - logApiError(this.config, { - model: this.model, - error: errorMessage, - status_code: statusCode, - error_type: errorType, - duration_ms: durationMs, - }); + logApiError( + this.config, + new ApiErrorEvent(this.model, errorMessage, durationMs, errorType), + ); } /** @@ -402,6 +389,17 @@ export class GeminiChat { this.history = history; } + getFinalUsageMetadata( + chunks: GenerateContentResponse[], + ): GenerateContentResponseUsageMetadata | undefined { + const lastChunkWithMetadata = chunks + .slice() + .reverse() + .find((chunk) => chunk.usageMetadata); + + return lastChunkWithMetadata?.usageMetadata; + } + private async *processStreamResponse( streamResponse: AsyncGenerator, inputContent: Content, @@ -444,7 +442,7 @@ export class GeminiChat { const fullText = getStructuredResponseFromParts(allParts); await this._logApiResponse( durationMs, - getFinalUsageMetadata(chunks), + this.getFinalUsageMetadata(chunks), fullText, ); } diff --git a/packages/core/src/core/nonInteractiveToolExecutor.test.ts b/packages/core/src/core/nonInteractiveToolExecutor.test.ts index edf11d35..c6874c6e 100644 --- a/packages/core/src/core/nonInteractiveToolExecutor.test.ts +++ b/packages/core/src/core/nonInteractiveToolExecutor.test.ts @@ -16,7 +16,10 @@ import { } from '../index.js'; import { Part, Type } from '@google/genai'; -const mockConfig = {} as unknown as Config; +const mockConfig = { + getSessionId: () => 'test-session-id', + getDisableDataCollection: () => false, +} as unknown as Config; describe('executeToolCall', () => { let mockToolRegistry: ToolRegistry; diff --git a/packages/core/src/core/nonInteractiveToolExecutor.ts b/packages/core/src/core/nonInteractiveToolExecutor.ts index f2174e06..8efb58e0 100644 --- a/packages/core/src/core/nonInteractiveToolExecutor.ts +++ b/packages/core/src/core/nonInteractiveToolExecutor.ts @@ -33,6 +33,8 @@ export async function executeToolCall( ); const durationMs = Date.now() - startTime; logToolCall(config, { + 'event.name': 'tool_call', + 'event.timestamp': new Date().toISOString(), function_name: toolCallRequest.name, function_args: toolCallRequest.args, duration_ms: durationMs, @@ -67,6 +69,8 @@ export async function executeToolCall( const durationMs = Date.now() - startTime; logToolCall(config, { + 'event.name': 'tool_call', + 'event.timestamp': new Date().toISOString(), function_name: toolCallRequest.name, function_args: toolCallRequest.args, duration_ms: durationMs, @@ -89,6 +93,8 @@ export async function executeToolCall( const error = e instanceof Error ? e : new Error(String(e)); const durationMs = Date.now() - startTime; logToolCall(config, { + 'event.name': 'tool_call', + 'event.timestamp': new Date().toISOString(), function_name: toolCallRequest.name, function_args: toolCallRequest.args, duration_ms: durationMs, diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts new file mode 100644 index 00000000..8da928c7 --- /dev/null +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -0,0 +1,338 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Buffer } from 'buffer'; +import * as https from 'https'; +import { + StartSessionEvent, + EndSessionEvent, + UserPromptEvent, + ToolCallEvent, + ApiRequestEvent, + ApiResponseEvent, + ApiErrorEvent, +} from '../types.js'; +import { EventMetadataKey } from './event-metadata-key.js'; +import { Config } from '../../config/config.js'; +import { getPersistentUserId } from '../../utils/user_id.js'; + +const start_session_event_name = 'start_session'; +const new_prompt_event_name = 'new_prompt'; +const tool_call_event_name = 'tool_call'; +const api_request_event_name = 'api_request'; +const api_response_event_name = 'api_response'; +const api_error_event_name = 'api_error'; +const end_session_event_name = 'end_session'; + +export interface LogResponse { + nextRequestWaitMs?: number; +} + +// Singleton class for batch posting log events to Clearcut. When a new event comes in, the elapsed time +// is checked and events are flushed to Clearcut if at least a minute has passed since the last flush. +export class ClearcutLogger { + private static instance: ClearcutLogger; + private config?: Config; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format. + private readonly events: any = []; + private last_flush_time: number = Date.now(); + private flush_interval_ms: number = 1000 * 60; // Wait at least a minute before flushing events. + + private constructor(config?: Config) { + this.config = config; + } + + static getInstance(config?: Config): ClearcutLogger | undefined { + if (config === undefined || config?.getDisableDataCollection()) + return undefined; + if (!ClearcutLogger.instance) { + ClearcutLogger.instance = new ClearcutLogger(config); + } + return ClearcutLogger.instance; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format. + enqueueLogEvent(event: any): void { + this.events.push([ + { + event_time_ms: Date.now(), + source_extension_json: JSON.stringify(event), + }, + ]); + } + + createLogEvent(name: string, data: Map): object { + return { + Application: 'GEMINI_CLI', + event_name: name, + client_install_id: getPersistentUserId(), + event_metadata: [data] as object[], + }; + } + + flushIfNeeded(): void { + if (Date.now() - this.last_flush_time < this.flush_interval_ms) { + return; + } + + this.flushToClearcut(); + this.last_flush_time = Date.now(); + } + + flushToClearcut(): Promise { + return new Promise((resolve, reject) => { + const request = [ + { + log_source_name: 'CONCORD', + request_time_ms: Date.now(), + log_event: this.events, + }, + ]; + const body = JSON.stringify(request); + const options = { + hostname: 'play.googleapis.com', + path: '/log', + method: 'POST', + headers: { 'Content-Length': Buffer.byteLength(body) }, + }; + const bufs: Buffer[] = []; + const req = https.request(options, (res) => { + res.on('data', (buf) => bufs.push(buf)); + res.on('end', () => { + resolve(Buffer.concat(bufs)); + }); + }); + req.on('error', (e) => { + reject(e); + }); + req.end(body); + }).then((buf: Buffer) => { + try { + this.events.length = 0; + return this.decodeLogResponse(buf) || {}; + } catch (error: unknown) { + console.error('Error flushing log events:', error); + return {}; + } + }); + } + + // Visible for testing. Decodes protobuf-encoded response from Clearcut server. + decodeLogResponse(buf: Buffer): LogResponse | undefined { + // TODO(obrienowen): return specific errors to facilitate debugging. + if (buf.length < 1) { + return undefined; + } + + // The first byte of the buffer is `field<<3 | type`. We're looking for field + // 1, with type varint, represented by type=0. If the first byte isn't 8, that + // means field 1 is missing or the message is corrupted. Either way, we return + // undefined. + if (buf.readUInt8(0) !== 8) { + return undefined; + } + + let ms = BigInt(0); + let cont = true; + + // In each byte, the most significant bit is the continuation bit. If it's + // set, we keep going. The lowest 7 bits, are data bits. They are concatenated + // in reverse order to form the final number. + for (let i = 1; cont && i < buf.length; i++) { + const byte = buf.readUInt8(i); + ms |= BigInt(byte & 0x7f) << BigInt(7 * (i - 1)); + cont = (byte & 0x80) !== 0; + } + + if (cont) { + // We have fallen off the buffer without seeing a terminating byte. The + // message is corrupted. + return undefined; + } + return { + nextRequestWaitMs: Number(ms), + }; + } + + logStartSessionEvent(event: StartSessionEvent): void { + const data: Map = new Map(); + + data.set(EventMetadataKey.GEMINI_CLI_START_SESSION_MODEL, event.model); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_EMBEDDING_MODEL, + event.embedding_model, + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_SANDBOX, + event.sandbox_enabled.toString(), + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_CORE_TOOLS, + event.core_tools_enabled, + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_APPROVAL_MODE, + event.approval_mode, + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_API_KEY_ENABLED, + event.api_key_enabled.toString(), + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED, + event.vertex_ai_enabled.toString(), + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED, + event.debug_enabled.toString(), + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_MCP_SERVERS, + event.mcp_servers, + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED, + event.telemetry_enabled.toString(), + ); + data.set( + EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED, + event.telemetry_log_user_prompts_enabled.toString(), + ); + + this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data)); + this.flushIfNeeded(); + } + + logNewPromptEvent(event: UserPromptEvent): void { + const data: Map = new Map(); + + data.set( + EventMetadataKey.GEMINI_CLI_USER_PROMPT_LENGTH, + JSON.stringify(event.prompt_length), + ); + + this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data)); + this.flushIfNeeded(); + } + + logToolCallEvent(event: ToolCallEvent): void { + const data: Map = new Map(); + + data.set(EventMetadataKey.GEMINI_CLI_TOOL_CALL_NAME, event.function_name); + data.set( + EventMetadataKey.GEMINI_CLI_TOOL_CALL_DECISION, + JSON.stringify(event.decision), + ); + data.set( + EventMetadataKey.GEMINI_CLI_TOOL_CALL_SUCCESS, + JSON.stringify(event.success), + ); + data.set( + EventMetadataKey.GEMINI_CLI_TOOL_CALL_DURATION_MS, + JSON.stringify(event.duration_ms), + ); + data.set( + EventMetadataKey.GEMINI_CLI_TOOL_ERROR_MESSAGE, + JSON.stringify(event.error), + ); + data.set( + EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE, + JSON.stringify(event.error_type), + ); + + this.enqueueLogEvent(this.createLogEvent(tool_call_event_name, data)); + this.flushIfNeeded(); + } + + logApiRequestEvent(event: ApiRequestEvent): void { + const data: Map = new Map(); + + data.set(EventMetadataKey.GEMINI_CLI_API_REQUEST_MODEL, event.model); + + this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data)); + this.flushIfNeeded(); + } + + logApiResponseEvent(event: ApiResponseEvent): void { + const data: Map = new Map(); + + data.set(EventMetadataKey.GEMINI_CLI_API_RESPONSE_MODEL, event.model); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_STATUS_CODE, + JSON.stringify(event.status_code), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_DURATION_MS, + JSON.stringify(event.duration_ms), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_ERROR_MESSAGE, + JSON.stringify(event.error), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT, + JSON.stringify(event.input_token_count), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT, + JSON.stringify(event.output_token_count), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT, + JSON.stringify(event.cached_content_token_count), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT, + JSON.stringify(event.thoughts_token_count), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT, + JSON.stringify(event.tool_token_count), + ); + + this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data)); + this.flushIfNeeded(); + } + + logApiErrorEvent(event: ApiErrorEvent): void { + const data: Map = new Map(); + + data.set(EventMetadataKey.GEMINI_CLI_API_ERROR_MODEL, event.model); + data.set( + EventMetadataKey.GEMINI_CLI_API_ERROR_TYPE, + JSON.stringify(event.error_type), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_ERROR_STATUS_CODE, + JSON.stringify(event.status_code), + ); + data.set( + EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS, + JSON.stringify(event.duration_ms), + ); + + this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data)); + this.flushIfNeeded(); + } + + logEndSessionEvent(event: EndSessionEvent): void { + const data: Map = new Map(); + + data.set( + EventMetadataKey.GEMINI_CLI_END_SESSION_ID, + event?.session_id?.toString() ?? '', + ); + + this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data)); + // Flush immediately on session end. + this.flushToClearcut(); + } + + shutdown() { + const event = new EndSessionEvent(this.config); + this.logEndSessionEvent(event); + } +} diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts new file mode 100644 index 00000000..146dcdeb --- /dev/null +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -0,0 +1,153 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +// Defines valid event metadata keys for Clearcut logging. +export enum EventMetadataKey { + GEMINI_CLI_KEY_UNKNOWN = 0, + + // ========================================================================== + // Start Session Event Keys + // =========================================================================== + + // Logs the model id used in the session. + GEMINI_CLI_START_SESSION_MODEL = 1, + + // Logs the embedding model id used in the session. + GEMINI_CLI_START_SESSION_EMBEDDING_MODEL = 2, + + // Logs the sandbox that was used in the session. + GEMINI_CLI_START_SESSION_SANDBOX = 3, + + // Logs the core tools that were enabled in the session. + GEMINI_CLI_START_SESSION_CORE_TOOLS = 4, + + // Logs the approval mode that was used in the session. + GEMINI_CLI_START_SESSION_APPROVAL_MODE = 5, + + // Logs whether an API key was used in the session. + GEMINI_CLI_START_SESSION_API_KEY_ENABLED = 6, + + // Logs whether the Vertex API was used in the session. + GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED = 7, + + // Logs whether debug mode was enabled in the session. + GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED = 8, + + // Logs the MCP servers that were enabled in the session. + GEMINI_CLI_START_SESSION_MCP_SERVERS = 9, + + // Logs whether user-collected telemetry was enabled in the session. + GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED = 10, + + // Logs whether prompt collection was enabled for user-collected telemetry. + GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED = 11, + + // Logs whether the session was configured to respect gitignore files. + GEMINI_CLI_START_SESSION_RESPECT_GITIGNORE = 12, + + // ========================================================================== + // User Prompt Event Keys + // =========================================================================== + + // Logs the length of the prompt. + GEMINI_CLI_USER_PROMPT_LENGTH = 13, + + // ========================================================================== + // Tool Call Event Keys + // =========================================================================== + + // Logs the function name. + GEMINI_CLI_TOOL_CALL_NAME = 14, + + // Logs the user's decision about how to handle the tool call. + GEMINI_CLI_TOOL_CALL_DECISION = 15, + + // Logs whether the tool call succeeded. + GEMINI_CLI_TOOL_CALL_SUCCESS = 16, + + // Logs the tool call duration in milliseconds. + GEMINI_CLI_TOOL_CALL_DURATION_MS = 17, + + // Logs the tool call error message, if any. + GEMINI_CLI_TOOL_ERROR_MESSAGE = 18, + + // Logs the tool call error type, if any. + GEMINI_CLI_TOOL_CALL_ERROR_TYPE = 19, + + // ========================================================================== + // GenAI API Request Event Keys + // =========================================================================== + + // Logs the model id of the request. + GEMINI_CLI_API_REQUEST_MODEL = 20, + + // ========================================================================== + // GenAI API Response Event Keys + // =========================================================================== + + // Logs the model id of the API call. + GEMINI_CLI_API_RESPONSE_MODEL = 21, + + // Logs the status code of the response. + GEMINI_CLI_API_RESPONSE_STATUS_CODE = 22, + + // Logs the duration of the API call in milliseconds. + GEMINI_CLI_API_RESPONSE_DURATION_MS = 23, + + // Logs the error message of the API call, if any. + GEMINI_CLI_API_ERROR_MESSAGE = 24, + + // Logs the input token count of the API call. + GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT = 25, + + // Logs the output token count of the API call. + GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT = 26, + + // Logs the cached token count of the API call. + GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT = 27, + + // Logs the thinking token count of the API call. + GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT = 28, + + // Logs the tool use token count of the API call. + GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT = 29, + + // ========================================================================== + // GenAI API Error Event Keys + // =========================================================================== + + // Logs the model id of the API call. + GEMINI_CLI_API_ERROR_MODEL = 30, + + // Logs the error type. + GEMINI_CLI_API_ERROR_TYPE = 31, + + // Logs the status code of the error response. + GEMINI_CLI_API_ERROR_STATUS_CODE = 32, + + // Logs the duration of the API call in milliseconds. + GEMINI_CLI_API_ERROR_DURATION_MS = 33, + + // ========================================================================== + // End Session Event Keys + // =========================================================================== + + // Logs the end of a session. + GEMINI_CLI_END_SESSION_ID = 34, +} + +export function getEventMetadataKey( + keyName: string, +): EventMetadataKey | undefined { + // Access the enum member by its string name + const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey]; + + // Check if the result is a valid enum member (not undefined and is a number) + if (typeof key === 'number') { + return key; + } + return undefined; +} diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index 6329b401..138c8486 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -25,15 +25,15 @@ export { logApiRequest, logApiError, logApiResponse, - getFinalUsageMetadata, } from './loggers.js'; export { + StartSessionEvent, + EndSessionEvent, UserPromptEvent, ToolCallEvent, ApiRequestEvent, ApiErrorEvent, ApiResponseEvent, - CliConfigEvent, TelemetryEvent, } from './types.js'; export { SpanStatusCode, ValueType } from '@opentelemetry/api'; diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 3fa9ad1c..2659f398 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -4,14 +4,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { ToolConfirmationOutcome } from '../tools/tools.js'; -import { AuthType } from '../core/contentGenerator.js'; +import { + AuthType, + CompletedToolCall, + ContentGeneratorConfig, + EditTool, + ErroredToolCall, + GeminiClient, + ToolConfirmationOutcome, + ToolRegistry, +} from '../index.js'; import { logs } from '@opentelemetry/api-logs'; import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import { Config } from '../config/config.js'; import { EVENT_API_REQUEST, EVENT_API_RESPONSE, + EVENT_CLI_CONFIG, + EVENT_TOOL_CALL, EVENT_USER_PROMPT, } from './constants.js'; import { @@ -20,13 +30,19 @@ import { logCliConfiguration, logUserPrompt, logToolCall, - ToolCallDecision, - getFinalUsageMetadata, } from './loggers.js'; +import { + ApiRequestEvent, + ApiResponseEvent, + StartSessionEvent, + ToolCallDecision, + ToolCallEvent, + UserPromptEvent, +} from './types.js'; import * as metrics from './metrics.js'; import * as sdk from './sdk.js'; import { vi, describe, beforeEach, it, expect } from 'vitest'; -import { GenerateContentResponse } from '@google/genai'; +import { GenerateContentResponseUsageMetadata } from '@google/genai'; describe('loggers', () => { const mockLogger = { @@ -54,8 +70,11 @@ describe('loggers', () => { apiKey: 'test-api-key', authType: AuthType.USE_VERTEX_AI, }), + getTelemetryEnabled: () => true, + getDisableDataCollection: () => false, getTelemetryLogPromptsEnabled: () => true, getFileFilteringRespectGitIgnore: () => true, + getFileFilteringAllowBuildArtifacts: () => false, getDebugMode: () => true, getMcpServers: () => ({ 'test-server': { @@ -63,15 +82,18 @@ describe('loggers', () => { }, }), getQuestion: () => 'test-question', + getTargetDir: () => 'target-dir', + getProxy: () => 'http://test.proxy.com:8080', } as unknown as Config; - logCliConfiguration(mockConfig); + const startSessionEvent = new StartSessionEvent(mockConfig); + logCliConfiguration(mockConfig, startSessionEvent); expect(mockLogger.emit).toHaveBeenCalledWith({ body: 'CLI configuration loaded.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.config', + 'event.name': EVENT_CLI_CONFIG, 'event.timestamp': '2025-01-01T00:00:00.000Z', model: 'test-model', embedding_model: 'test-embedding-model', @@ -92,14 +114,13 @@ describe('loggers', () => { describe('logUserPrompt', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, getTelemetryLogPromptsEnabled: () => true, + getDisableDataCollection: () => false, } as unknown as Config; it('should log a user prompt', () => { - const event = { - prompt: 'test-prompt', - prompt_length: 11, - }; + const event = new UserPromptEvent(11, 'test-prompt'); logUserPrompt(mockConfig, event); @@ -118,12 +139,12 @@ describe('loggers', () => { it('should not log prompt if disabled', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, getTelemetryLogPromptsEnabled: () => false, + getTargetDir: () => 'target-dir', + getDisableDataCollection: () => false, } as unknown as Config; - const event = { - prompt: 'test-prompt', - prompt_length: 11, - }; + const event = new UserPromptEvent(11, 'test-prompt'); logUserPrompt(mockConfig, event); @@ -142,6 +163,10 @@ describe('loggers', () => { describe('logApiResponse', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getDisableDataCollection: () => false, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, } as Config; const mockMetrics = { @@ -159,17 +184,19 @@ describe('loggers', () => { }); it('should log an API response with all fields', () => { - const event = { - model: 'test-model', - status_code: 200, - duration_ms: 100, - input_token_count: 17, - output_token_count: 50, - cached_content_token_count: 10, - thoughts_token_count: 5, - tool_token_count: 2, - response_text: 'test-response', + const usageData: GenerateContentResponseUsageMetadata = { + promptTokenCount: 17, + candidatesTokenCount: 50, + cachedContentTokenCount: 10, + thoughtsTokenCount: 5, + toolUsePromptTokenCount: 2, }; + const event = new ApiResponseEvent( + 'test-model', + 100, + usageData, + 'test-response', + ); logApiResponse(mockConfig, event); @@ -209,22 +236,25 @@ describe('loggers', () => { }); it('should log an API response with an error', () => { - const event = { - model: 'test-model', - duration_ms: 100, - error: 'test-error', - input_token_count: 17, - output_token_count: 50, - cached_content_token_count: 10, - thoughts_token_count: 5, - tool_token_count: 2, - response_text: 'test-response', + const usageData: GenerateContentResponseUsageMetadata = { + promptTokenCount: 17, + candidatesTokenCount: 50, + cachedContentTokenCount: 10, + thoughtsTokenCount: 5, + toolUsePromptTokenCount: 2, }; + const event = new ApiResponseEvent( + 'test-model', + 100, + usageData, + 'test-response', + 'test-error', + ); logApiResponse(mockConfig, event); expect(mockLogger.emit).toHaveBeenCalledWith({ - body: 'API response from test-model. Status: N/A. Duration: 100ms.', + body: 'API response from test-model. Status: 200. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', ...event, @@ -239,13 +269,14 @@ describe('loggers', () => { describe('logApiRequest', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getDisableDataCollection: () => false, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, } as Config; it('should log an API request with request_text', () => { - const event = { - model: 'test-model', - request_text: 'This is a test request', - }; + const event = new ApiRequestEvent('test-model', 'This is a test request'); logApiRequest(mockConfig, event); @@ -262,9 +293,7 @@ describe('loggers', () => { }); it('should log an API request without request_text', () => { - const event = { - model: 'test-model', - }; + const event = new ApiRequestEvent('test-model'); logApiRequest(mockConfig, event); @@ -281,8 +310,46 @@ describe('loggers', () => { }); describe('logToolCall', () => { + const cfg1 = { + getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getGeminiClient: () => mockGeminiClient, + } as Config; + const cfg2 = { + getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getProxy: () => 'http://test.proxy.com:8080', + getContentGeneratorConfig: () => + ({ model: 'test-model' }) as ContentGeneratorConfig, + getModel: () => 'test-model', + getEmbeddingModel: () => 'test-embedding-model', + getWorkingDir: () => 'test-working-dir', + getSandbox: () => true, + getCoreTools: () => ['ls', 'read-file'], + getApprovalMode: () => 'default', + getTelemetryLogPromptsEnabled: () => true, + getFileFilteringRespectGitIgnore: () => true, + getFileFilteringAllowBuildArtifacts: () => false, + getDebugMode: () => true, + getMcpServers: () => ({ + 'test-server': { + command: 'test-command', + }, + }), + getQuestion: () => 'test-question', + getToolRegistry: () => new ToolRegistry(cfg1), + getFullContext: () => false, + getUserMemory: () => 'user-memory', + } as unknown as Config; + + const mockGeminiClient = new GeminiClient(cfg2); const mockConfig = { getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getGeminiClient: () => mockGeminiClient, + getDisableDataCollection: () => false, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, } as Config; const mockMetrics = { @@ -297,23 +364,36 @@ describe('loggers', () => { }); it('should log a tool call with all fields', () => { - const event = { - function_name: 'test-function', - function_args: { - arg1: 'value1', - arg2: 2, + const call: CompletedToolCall = { + status: 'success', + request: { + name: 'test-function', + args: { + arg1: 'value1', + arg2: 2, + }, + callId: 'test-call-id', + isClientInitiated: true, }, - duration_ms: 100, - success: true, + response: { + callId: 'test-call-id', + responseParts: 'test-response', + resultDisplay: undefined, + error: undefined, + }, + tool: new EditTool(mockConfig), + durationMs: 100, + outcome: ToolConfirmationOutcome.ProceedOnce, }; + const event = new ToolCallEvent(call); - logToolCall(mockConfig, event, ToolConfirmationOutcome.ProceedOnce); + logToolCall(mockConfig, event); expect(mockLogger.emit).toHaveBeenCalledWith({ body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.tool_call', + 'event.name': EVENT_TOOL_CALL, 'event.timestamp': '2025-01-01T00:00:00.000Z', function_name: 'test-function', function_args: JSON.stringify( @@ -339,23 +419,35 @@ describe('loggers', () => { ); }); it('should log a tool call with a reject decision', () => { - const event = { - function_name: 'test-function', - function_args: { - arg1: 'value1', - arg2: 2, + const call: ErroredToolCall = { + status: 'error', + request: { + name: 'test-function', + args: { + arg1: 'value1', + arg2: 2, + }, + callId: 'test-call-id', + isClientInitiated: true, }, - duration_ms: 100, - success: false, + response: { + callId: 'test-call-id', + responseParts: 'test-response', + resultDisplay: undefined, + error: undefined, + }, + durationMs: 100, + outcome: ToolConfirmationOutcome.Cancel, }; + const event = new ToolCallEvent(call); - logToolCall(mockConfig, event, ToolConfirmationOutcome.Cancel); + logToolCall(mockConfig, event); expect(mockLogger.emit).toHaveBeenCalledWith({ body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.tool_call', + 'event.name': EVENT_TOOL_CALL, 'event.timestamp': '2025-01-01T00:00:00.000Z', function_name: 'test-function', function_args: JSON.stringify( @@ -382,23 +474,36 @@ describe('loggers', () => { }); it('should log a tool call with a modify decision', () => { - const event = { - function_name: 'test-function', - function_args: { - arg1: 'value1', - arg2: 2, + const call: CompletedToolCall = { + status: 'success', + request: { + name: 'test-function', + args: { + arg1: 'value1', + arg2: 2, + }, + callId: 'test-call-id', + isClientInitiated: true, }, - duration_ms: 100, - success: true, + response: { + callId: 'test-call-id', + responseParts: 'test-response', + resultDisplay: undefined, + error: undefined, + }, + outcome: ToolConfirmationOutcome.ModifyWithEditor, + tool: new EditTool(mockConfig), + durationMs: 100, }; + const event = new ToolCallEvent(call); - logToolCall(mockConfig, event, ToolConfirmationOutcome.ModifyWithEditor); + logToolCall(mockConfig, event); expect(mockLogger.emit).toHaveBeenCalledWith({ body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.tool_call', + 'event.name': EVENT_TOOL_CALL, 'event.timestamp': '2025-01-01T00:00:00.000Z', function_name: 'test-function', function_args: JSON.stringify( @@ -425,15 +530,27 @@ describe('loggers', () => { }); it('should log a tool call without a decision', () => { - const event = { - function_name: 'test-function', - function_args: { - arg1: 'value1', - arg2: 2, + const call: CompletedToolCall = { + status: 'success', + request: { + name: 'test-function', + args: { + arg1: 'value1', + arg2: 2, + }, + callId: 'test-call-id', + isClientInitiated: true, }, - duration_ms: 100, - success: true, + response: { + callId: 'test-call-id', + responseParts: 'test-response', + resultDisplay: undefined, + error: undefined, + }, + tool: new EditTool(mockConfig), + durationMs: 100, }; + const event = new ToolCallEvent(call); logToolCall(mockConfig, event); @@ -441,7 +558,7 @@ describe('loggers', () => { body: 'Tool call: test-function. Success: true. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.tool_call', + 'event.name': EVENT_TOOL_CALL, 'event.timestamp': '2025-01-01T00:00:00.000Z', function_name: 'test-function', function_args: JSON.stringify( @@ -467,17 +584,29 @@ describe('loggers', () => { }); it('should log a failed tool call with an error', () => { - const event = { - function_name: 'test-function', - function_args: { - arg1: 'value1', - arg2: 2, + const call: ErroredToolCall = { + status: 'error', + request: { + name: 'test-function', + args: { + arg1: 'value1', + arg2: 2, + }, + callId: 'test-call-id', + isClientInitiated: true, }, - duration_ms: 100, - success: false, - error: 'test-error', - error_type: 'test-error-type', + response: { + callId: 'test-call-id', + responseParts: 'test-response', + resultDisplay: undefined, + error: { + name: 'test-error-type', + message: 'test-error', + }, + }, + durationMs: 100, }; + const event = new ToolCallEvent(call); logToolCall(mockConfig, event); @@ -485,7 +614,7 @@ describe('loggers', () => { body: 'Tool call: test-function. Success: false. Duration: 100ms.', attributes: { 'session.id': 'test-session-id', - 'event.name': 'gemini_cli.tool_call', + 'event.name': EVENT_TOOL_CALL, 'event.timestamp': '2025-01-01T00:00:00.000Z', function_name: 'test-function', function_args: JSON.stringify( @@ -515,75 +644,3 @@ describe('loggers', () => { }); }); }); - -describe('getFinalUsageMetadata', () => { - const createMockResponse = ( - usageMetadata?: GenerateContentResponse['usageMetadata'], - ): GenerateContentResponse => - ({ - text: () => '', - data: () => ({}) as Record, - functionCalls: () => [], - executableCode: () => [], - codeExecutionResult: () => [], - usageMetadata, - }) as unknown as GenerateContentResponse; - - it('should return the usageMetadata from the last chunk that has it', () => { - const chunks: GenerateContentResponse[] = [ - createMockResponse({ - promptTokenCount: 10, - candidatesTokenCount: 20, - totalTokenCount: 30, - }), - createMockResponse(), - createMockResponse({ - promptTokenCount: 15, - candidatesTokenCount: 25, - totalTokenCount: 40, - }), - createMockResponse(), - ]; - - const result = getFinalUsageMetadata(chunks); - expect(result).toEqual({ - promptTokenCount: 15, - candidatesTokenCount: 25, - totalTokenCount: 40, - }); - }); - - it('should return undefined if no chunks have usageMetadata', () => { - const chunks: GenerateContentResponse[] = [ - createMockResponse(), - createMockResponse(), - createMockResponse(), - ]; - - const result = getFinalUsageMetadata(chunks); - expect(result).toBeUndefined(); - }); - - it('should return the metadata from the only chunk if it has it', () => { - const chunks: GenerateContentResponse[] = [ - createMockResponse({ - promptTokenCount: 1, - candidatesTokenCount: 2, - totalTokenCount: 3, - }), - ]; - - const result = getFinalUsageMetadata(chunks); - expect(result).toEqual({ - promptTokenCount: 1, - candidatesTokenCount: 2, - totalTokenCount: 3, - }); - }); - - it('should return undefined for an empty array of chunks', () => { - const chunks: GenerateContentResponse[] = []; - const result = getFinalUsageMetadata(chunks); - expect(result).toBeUndefined(); - }); -}); diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index 0ecf130f..054386b8 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -20,6 +20,7 @@ import { ApiErrorEvent, ApiRequestEvent, ApiResponseEvent, + StartSessionEvent, ToolCallEvent, UserPromptEvent, } from './types.js'; @@ -30,15 +31,10 @@ import { recordToolCallMetrics, } from './metrics.js'; import { isTelemetrySdkInitialized } from './sdk.js'; -import { ToolConfirmationOutcome } from '../tools/tools.js'; -import { - GenerateContentResponse, - GenerateContentResponseUsageMetadata, -} from '@google/genai'; -import { AuthType } from '../core/contentGenerator.js'; +import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js'; const shouldLogUserPrompts = (config: Config): boolean => - config.getTelemetryLogPromptsEnabled() ?? false; + config.getTelemetryLogPromptsEnabled(); function getCommonAttributes(config: Config): LogAttributes { return { @@ -46,59 +42,30 @@ function getCommonAttributes(config: Config): LogAttributes { }; } -export enum ToolCallDecision { - ACCEPT = 'accept', - REJECT = 'reject', - MODIFY = 'modify', -} - -export function getDecisionFromOutcome( - outcome: ToolConfirmationOutcome, -): ToolCallDecision { - switch (outcome) { - case ToolConfirmationOutcome.ProceedOnce: - case ToolConfirmationOutcome.ProceedAlways: - case ToolConfirmationOutcome.ProceedAlwaysServer: - case ToolConfirmationOutcome.ProceedAlwaysTool: - return ToolCallDecision.ACCEPT; - case ToolConfirmationOutcome.ModifyWithEditor: - return ToolCallDecision.MODIFY; - case ToolConfirmationOutcome.Cancel: - default: - return ToolCallDecision.REJECT; - } -} - -export function logCliConfiguration(config: Config): void { +export function logCliConfiguration( + config: Config, + event: StartSessionEvent, +): void { + ClearcutLogger.getInstance(config)?.logStartSessionEvent(event); if (!isTelemetrySdkInitialized()) return; - const generatorConfig = config.getContentGeneratorConfig(); - let useGemini = false; - let useVertex = false; - - if (generatorConfig && generatorConfig.authType) { - useGemini = generatorConfig.authType === AuthType.USE_GEMINI; - useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI; - } - - const mcpServers = config.getMcpServers(); const attributes: LogAttributes = { ...getCommonAttributes(config), 'event.name': EVENT_CLI_CONFIG, 'event.timestamp': new Date().toISOString(), - model: config.getModel(), - embedding_model: config.getEmbeddingModel(), - sandbox_enabled: !!config.getSandbox(), - core_tools_enabled: (config.getCoreTools() ?? []).join(','), - approval_mode: config.getApprovalMode(), - api_key_enabled: useGemini || useVertex, - vertex_ai_enabled: useVertex, - log_user_prompts_enabled: config.getTelemetryLogPromptsEnabled(), - file_filtering_respect_git_ignore: - config.getFileFilteringRespectGitIgnore(), - debug_mode: config.getDebugMode(), - mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '', + model: event.model, + embedding_model: event.embedding_model, + sandbox_enabled: event.sandbox_enabled, + core_tools_enabled: event.core_tools_enabled, + approval_mode: event.approval_mode, + api_key_enabled: event.api_key_enabled, + vertex_ai_enabled: event.vertex_ai_enabled, + log_user_prompts_enabled: event.telemetry_log_user_prompts_enabled, + file_filtering_respect_git_ignore: event.file_filtering_respect_git_ignore, + debug_mode: event.debug_enabled, + mcp_servers: event.mcp_servers, }; + const logger = logs.getLogger(SERVICE_NAME); const logRecord: LogRecord = { body: 'CLI configuration loaded.', @@ -107,12 +74,8 @@ export function logCliConfiguration(config: Config): void { logger.emit(logRecord); } -export function logUserPrompt( - config: Config, - event: Omit & { - prompt: string; - }, -): void { +export function logUserPrompt(config: Config, event: UserPromptEvent): void { + ClearcutLogger.getInstance(config)?.logNewPromptEvent(event); if (!isTelemetrySdkInitialized()) return; const attributes: LogAttributes = { @@ -134,22 +97,16 @@ export function logUserPrompt( logger.emit(logRecord); } -export function logToolCall( - config: Config, - event: Omit, - outcome?: ToolConfirmationOutcome, -): void { +export function logToolCall(config: Config, event: ToolCallEvent): void { + ClearcutLogger.getInstance(config)?.logToolCallEvent(event); if (!isTelemetrySdkInitialized()) return; - const decision = outcome ? getDecisionFromOutcome(outcome) : undefined; - const attributes: LogAttributes = { ...getCommonAttributes(config), ...event, 'event.name': EVENT_TOOL_CALL, 'event.timestamp': new Date().toISOString(), function_args: JSON.stringify(event.function_args, null, 2), - decision, }; if (event.error) { attributes['error.message'] = event.error; @@ -157,9 +114,10 @@ export function logToolCall( attributes['error.type'] = event.error_type; } } + const logger = logs.getLogger(SERVICE_NAME); const logRecord: LogRecord = { - body: `Tool call: ${event.function_name}${decision ? `. Decision: ${decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`, + body: `Tool call: ${event.function_name}${event.decision ? `. Decision: ${event.decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`, attributes, }; logger.emit(logRecord); @@ -168,21 +126,21 @@ export function logToolCall( event.function_name, event.duration_ms, event.success, - decision, + event.decision, ); } -export function logApiRequest( - config: Config, - event: Omit, -): void { +export function logApiRequest(config: Config, event: ApiRequestEvent): void { + ClearcutLogger.getInstance(config)?.logApiRequestEvent(event); if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { ...getCommonAttributes(config), ...event, 'event.name': EVENT_API_REQUEST, 'event.timestamp': new Date().toISOString(), }; + const logger = logs.getLogger(SERVICE_NAME); const logRecord: LogRecord = { body: `API request to ${event.model}.`, @@ -191,17 +149,18 @@ export function logApiRequest( logger.emit(logRecord); } -export function logApiError( - config: Config, - event: Omit, -): void { +export function logApiError(config: Config, event: ApiErrorEvent): void { + ClearcutLogger.getInstance(config)?.logApiErrorEvent(event); if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { ...getCommonAttributes(config), ...event, 'event.name': EVENT_API_ERROR, 'event.timestamp': new Date().toISOString(), ['error.message']: event.error, + model_name: event.model, + duration: event.duration_ms, }; if (event.error_type) { @@ -226,10 +185,8 @@ export function logApiError( ); } -export function logApiResponse( - config: Config, - event: Omit, -): void { +export function logApiResponse(config: Config, event: ApiResponseEvent): void { + ClearcutLogger.getInstance(config)?.logApiResponseEvent(event); if (!isTelemetrySdkInitialized()) return; const attributes: LogAttributes = { ...getCommonAttributes(config), @@ -287,15 +244,3 @@ export function logApiResponse( ); recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool'); } - -export function getFinalUsageMetadata( - chunks: GenerateContentResponse[], -): GenerateContentResponseUsageMetadata | undefined { - // Only the last streamed item has the final token count. - const lastChunkWithMetadata = chunks - .slice() - .reverse() - .find((chunk) => chunk.usageMetadata); - - return lastChunkWithMetadata?.usageMetadata; -} diff --git a/packages/core/src/telemetry/sdk.ts b/packages/core/src/telemetry/sdk.ts index 61f501a6..033a9d77 100644 --- a/packages/core/src/telemetry/sdk.ts +++ b/packages/core/src/telemetry/sdk.ts @@ -29,6 +29,8 @@ import { Config } from '../config/config.js'; import { SERVICE_NAME } from './constants.js'; import { initializeMetrics } from './metrics.js'; import { logCliConfiguration } from './loggers.js'; +import { StartSessionEvent } from './types.js'; +import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js'; // For troubleshooting, set the log level to DiagLogLevel.DEBUG diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); @@ -113,7 +115,7 @@ export function initializeTelemetry(config: Config): void { console.log('OpenTelemetry SDK started successfully.'); telemetryInitialized = true; initializeMetrics(config); - logCliConfiguration(config); + logCliConfiguration(config, new StartSessionEvent(config)); } catch (error) { console.error('Error starting OpenTelemetry SDK:', error); } @@ -127,6 +129,7 @@ export async function shutdownTelemetry(): Promise { return; } try { + ClearcutLogger.getInstance()?.shutdown(); await sdk.shutdown(); console.log('OpenTelemetry SDK shut down successfully.'); } catch (error) { diff --git a/packages/core/src/telemetry/telemetry.test.ts b/packages/core/src/telemetry/telemetry.test.ts index 97c96c64..624c9ded 100644 --- a/packages/core/src/telemetry/telemetry.test.ts +++ b/packages/core/src/telemetry/telemetry.test.ts @@ -13,6 +13,7 @@ import { import { Config } from '../config/config.js'; import { NodeSDK } from '@opentelemetry/sdk-node'; import * as loggers from './loggers.js'; +import { StartSessionEvent } from './types.js'; vi.mock('@opentelemetry/sdk-node'); vi.mock('../config/config.js'); @@ -55,10 +56,11 @@ describe('telemetry', () => { it('should initialize the telemetry service', () => { initializeTelemetry(mockConfig); + const event = new StartSessionEvent(mockConfig); expect(NodeSDK).toHaveBeenCalled(); expect(mockNodeSdk.start).toHaveBeenCalled(); - expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig); + expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig, event); }); it('should shutdown the telemetry service', async () => { diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 68dd411e..f70daa78 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -4,16 +4,108 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { ToolCallDecision } from './loggers.js'; +import { GenerateContentResponseUsageMetadata } from '@google/genai'; +import { Config } from '../config/config.js'; +import { CompletedToolCall } from '../core/coreToolScheduler.js'; +import { ToolConfirmationOutcome } from '../tools/tools.js'; +import { AuthType } from '../core/contentGenerator.js'; -export interface UserPromptEvent { +export enum ToolCallDecision { + ACCEPT = 'accept', + REJECT = 'reject', + MODIFY = 'modify', +} + +export function getDecisionFromOutcome( + outcome: ToolConfirmationOutcome, +): ToolCallDecision { + switch (outcome) { + case ToolConfirmationOutcome.ProceedOnce: + case ToolConfirmationOutcome.ProceedAlways: + case ToolConfirmationOutcome.ProceedAlwaysServer: + case ToolConfirmationOutcome.ProceedAlwaysTool: + return ToolCallDecision.ACCEPT; + case ToolConfirmationOutcome.ModifyWithEditor: + return ToolCallDecision.MODIFY; + case ToolConfirmationOutcome.Cancel: + default: + return ToolCallDecision.REJECT; + } +} + +export class StartSessionEvent { + 'event.name': 'cli_config'; + 'event.timestamp': string; // ISO 8601 + model: string; + embedding_model: string; + sandbox_enabled: boolean; + core_tools_enabled: string; + approval_mode: string; + api_key_enabled: boolean; + vertex_ai_enabled: boolean; + debug_enabled: boolean; + mcp_servers: string; + telemetry_enabled: boolean; + telemetry_log_user_prompts_enabled: boolean; + file_filtering_respect_git_ignore: boolean; + + constructor(config: Config) { + const generatorConfig = config.getContentGeneratorConfig(); + const mcpServers = config.getMcpServers(); + + let useGemini = false; + let useVertex = false; + if (generatorConfig && generatorConfig.authType) { + useGemini = generatorConfig.authType === AuthType.USE_GEMINI; + useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI; + } + + this['event.name'] = 'cli_config'; + this.model = config.getModel(); + this.embedding_model = config.getEmbeddingModel(); + this.sandbox_enabled = + typeof config.getSandbox() === 'string' || !!config.getSandbox(); + this.core_tools_enabled = (config.getCoreTools() ?? []).join(','); + this.approval_mode = config.getApprovalMode(); + this.api_key_enabled = useGemini || useVertex; + this.vertex_ai_enabled = useVertex; + this.debug_enabled = config.getDebugMode(); + this.mcp_servers = mcpServers ? Object.keys(mcpServers).join(',') : ''; + this.telemetry_enabled = config.getTelemetryEnabled(); + this.telemetry_log_user_prompts_enabled = + config.getTelemetryLogPromptsEnabled(); + this.file_filtering_respect_git_ignore = + config.getFileFilteringRespectGitIgnore(); + } +} + +export class EndSessionEvent { + 'event.name': 'end_session'; + 'event.timestamp': string; // ISO 8601 + session_id?: string; + + constructor(config?: Config) { + this['event.name'] = 'end_session'; + this['event.timestamp'] = new Date().toISOString(); + this.session_id = config?.getSessionId(); + } +} + +export class UserPromptEvent { 'event.name': 'user_prompt'; 'event.timestamp': string; // ISO 8601 prompt_length: number; prompt?: string; + + constructor(prompt_length: number, prompt?: string) { + this['event.name'] = 'user_prompt'; + this['event.timestamp'] = new Date().toISOString(); + this.prompt_length = prompt_length; + this.prompt = prompt; + } } -export interface ToolCallEvent { +export class ToolCallEvent { 'event.name': 'tool_call'; 'event.timestamp': string; // ISO 8601 function_name: string; @@ -23,16 +115,37 @@ export interface ToolCallEvent { decision?: ToolCallDecision; error?: string; error_type?: string; + + constructor(call: CompletedToolCall) { + this['event.name'] = 'tool_call'; + this['event.timestamp'] = new Date().toISOString(); + this.function_name = call.request.name; + this.function_args = call.request.args; + this.duration_ms = call.durationMs ?? 0; + this.success = call.status === 'success'; + this.decision = call.outcome + ? getDecisionFromOutcome(call.outcome) + : undefined; + this.error = call.response.error?.message; + this.error_type = call.response.error?.name; + } } -export interface ApiRequestEvent { +export class ApiRequestEvent { 'event.name': 'api_request'; 'event.timestamp': string; // ISO 8601 model: string; request_text?: string; + + constructor(model: string, request_text?: string) { + this['event.name'] = 'api_request'; + this['event.timestamp'] = new Date().toISOString(); + this.model = model; + this.request_text = request_text; + } } -export interface ApiErrorEvent { +export class ApiErrorEvent { 'event.name': 'api_error'; 'event.timestamp': string; // ISO 8601 model: string; @@ -40,9 +153,25 @@ export interface ApiErrorEvent { error_type?: string; status_code?: number | string; duration_ms: number; + + constructor( + model: string, + error: string, + duration_ms: number, + error_type?: string, + status_code?: number | string, + ) { + this['event.name'] = 'api_error'; + this['event.timestamp'] = new Date().toISOString(); + this.model = model; + this.error = error; + this.error_type = error_type; + this.status_code = status_code; + this.duration_ms = duration_ms; + } } -export interface ApiResponseEvent { +export class ApiResponseEvent { 'event.name': 'api_response'; 'event.timestamp': string; // ISO 8601 model: string; @@ -55,24 +184,34 @@ export interface ApiResponseEvent { thoughts_token_count: number; tool_token_count: number; response_text?: string; -} -export interface CliConfigEvent { - 'event.name': 'cli_config'; - 'event.timestamp': string; // ISO 8601 - model: string; - sandbox_enabled: boolean; - core_tools_enabled: string; - approval_mode: string; - vertex_ai_enabled: boolean; - log_user_prompts_enabled: boolean; - file_filtering_respect_git_ignore: boolean; + constructor( + model: string, + duration_ms: number, + usage_data?: GenerateContentResponseUsageMetadata, + response_text?: string, + error?: string, + ) { + this['event.name'] = 'api_response'; + this['event.timestamp'] = new Date().toISOString(); + this.model = model; + this.duration_ms = duration_ms; + this.status_code = 200; + this.input_token_count = usage_data?.promptTokenCount ?? 0; + this.output_token_count = usage_data?.candidatesTokenCount ?? 0; + this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0; + this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0; + this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0; + this.response_text = response_text; + this.error = error; + } } export type TelemetryEvent = + | StartSessionEvent + | EndSessionEvent | UserPromptEvent | ToolCallEvent | ApiRequestEvent | ApiErrorEvent - | ApiResponseEvent - | CliConfigEvent; + | ApiResponseEvent; diff --git a/packages/core/src/utils/user_id.ts b/packages/core/src/utils/user_id.ts new file mode 100644 index 00000000..5db080a4 --- /dev/null +++ b/packages/core/src/utils/user_id.ts @@ -0,0 +1,58 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as os from 'os'; +import * as fs from 'fs'; +import * as path from 'path'; +import { randomUUID } from 'crypto'; +import { GEMINI_DIR } from './paths.js'; + +const homeDir = os.homedir() ?? ''; +const geminiDir = path.join(homeDir, GEMINI_DIR); +const userIdFile = path.join(geminiDir, 'user_id'); + +function ensureGeminiDirExists() { + if (!fs.existsSync(geminiDir)) { + fs.mkdirSync(geminiDir, { recursive: true }); + } +} + +function readUserIdFromFile(): string | null { + if (fs.existsSync(userIdFile)) { + const userId = fs.readFileSync(userIdFile, 'utf-8').trim(); + return userId || null; + } + return null; +} + +function writeUserIdToFile(userId: string) { + fs.writeFileSync(userIdFile, userId, 'utf-8'); +} + +/** + * Retrieves the persistent user ID from a file, creating it if it doesn't exist. + * This ID is used for unique user tracking. + * @returns A UUID string for the user. + */ +export function getPersistentUserId(): string { + try { + ensureGeminiDirExists(); + let userId = readUserIdFromFile(); + + if (!userId) { + userId = randomUUID(); + writeUserIdToFile(userId); + } + + return userId; + } catch (error) { + console.error( + 'Error accessing persistent user ID file, generating ephemeral ID:', + error, + ); + return '123456789'; + } +}