jcarr
/
gemini-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clearcut logging - initial implementation (#1274) 

Flag-guarded initial implementation of a clearcut logger to collect telemetry data and send it to Concord for dashboards, etc.
This commit is contained in:
owenofbrien 2025-06-22 09:26:48 -05:00 committed by GitHub
parent c9950b3cb2
commit 4cfab0a893
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 1051 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
packages/cli/src/config/config.test.ts
View File

@ -11,13 +11,11 @@ import { Settings } from './settings.js';
import { Extension } from './extension.js';
import * as ServerConfig from '@gemini-cli/core';
const MOCK_HOME_DIR = '/mock/home/user';
vi.mock('os', async (importOriginal) => {
const actualOs = await importOriginal<typeof os>();
return {
...actualOs,
homedir: vi.fn(() => MOCK_HOME_DIR),
homedir: vi.fn(() => '/mock/home/user'),
};
});
@ -53,7 +51,7 @@ describe('loadCliConfig', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
process.env.GEMINI_API_KEY = 'test-api-key'; // Ensure API key is set for tests
});
@ -98,7 +96,7 @@ describe('loadCliConfig telemetry', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
process.env.GEMINI_API_KEY = 'test-api-key';
});
@ -250,7 +248,7 @@ describe('loadCliConfig telemetry', () => {
describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
// Other common mocks would be reset here.
});
@ -310,7 +308,7 @@ describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => {
// Example of a previously failing test structure:
/*
it('should correctly use mocked homedir for global path', async () => {
const MOCK_GEMINI_DIR_LOCAL = path.join(MOCK_HOME_DIR, '.gemini');
const MOCK_GEMINI_DIR_LOCAL = path.join('/mock/home/user', '.gemini');
const MOCK_GLOBAL_PATH_LOCAL = path.join(MOCK_GEMINI_DIR_LOCAL, 'GEMINI.md');
mockFs({
[MOCK_GLOBAL_PATH_LOCAL]: { type: 'file', content: 'GlobalContentOnly' }

1
packages/cli/src/config/config.ts
View File

@ -226,6 +226,7 @@ export async function loadCliConfig(
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??
settings.telemetry?.otlpEndpoint,
logPrompts: argv.telemetryLogPrompts ?? settings.telemetry?.logPrompts,
disableDataCollection: settings.telemetry?.disableDataCollection ?? true,
},
// Git-aware file filtering settings
fileFiltering: {

8
packages/cli/src/config/settings.test.ts
View File

@ -6,15 +6,13 @@
/// <reference types="vitest/globals" />
const MOCK_HOME_DIR = '/mock/home/user'; // MUST BE FIRST
// Mock 'os' first. Its factory uses MOCK_HOME_DIR.
// Mock 'os' first.
import * as osActual from 'os'; // Import for type info for the mock factory
vi.mock('os', async (importOriginal) => {
const actualOs = await importOriginal<typeof osActual>();
return {
...actualOs,
homedir: vi.fn(() => MOCK_HOME_DIR),
homedir: vi.fn(() => '/mock/home/user'),
};
});
@ -77,7 +75,7 @@ describe('Settings Loading and Merging', () => {
mockFsMkdirSync = vi.mocked(fs.mkdirSync);
mockStripJsonComments = vi.mocked(stripJsonComments);
vi.mocked(osActual.homedir).mockReturnValue(MOCK_HOME_DIR);
vi.mocked(osActual.homedir).mockReturnValue('/mock/home/user');
(mockStripJsonComments as unknown as Mock).mockImplementation(
(jsonString: string) => jsonString,
);

2
packages/cli/src/gemini.tsx
View File

@ -131,6 +131,8 @@ export async function main() {
}
logUserPrompt(config, {
'event.name': 'user_prompt',
'event.timestamp': new Date().toISOString(),
prompt: input,
prompt_length: input.length,
});

6
packages/cli/src/ui/hooks/useGeminiStream.test.tsx
View File

@ -38,12 +38,17 @@ const MockedGeminiClientClass = vi.hoisted(() =>
}),
);
const MockedUserPromptEvent = vi.hoisted(() =>
vi.fn().mockImplementation(() => {}),
);
vi.mock('@gemini-cli/core', async (importOriginal) => {
const actualCoreModule = (await importOriginal()) as any;
return {
...actualCoreModule,
GitService: vi.fn(),
GeminiClient: MockedGeminiClientClass,
UserPromptEvent: MockedUserPromptEvent,
};
});
@ -283,6 +288,7 @@ describe('useGeminiStream', () => {
getProjectRoot: vi.fn(() => '/test/dir'),
getCheckpointingEnabled: vi.fn(() => false),
getGeminiClient: mockGetGeminiClient,
getDisableDataCollection: () => false,
addHistory: vi.fn(),
} as unknown as Config;
mockOnDebugMessage = vi.fn();

9
packages/cli/src/ui/hooks/useGeminiStream.ts
View File

@ -23,6 +23,7 @@ import {
EditorType,
ThoughtSummary,
isAuthError,
UserPromptEvent,
} from '@gemini-cli/core';
import { type Part, type PartListUnion } from '@google/genai';
import {
@ -213,10 +214,10 @@ export const useGeminiStream = (
if (typeof query === 'string') {
const trimmedQuery = query.trim();
logUserPrompt(config, {
prompt: trimmedQuery,
prompt_length: trimmedQuery.length,
});
logUserPrompt(
config,
new UserPromptEvent(trimmedQuery.length, trimmedQuery),
);
onDebugMessage(`User query: '${trimmedQuery}'`);
await logger?.logMessage(MessageSenderType.USER, trimmedQuery);

1
packages/cli/src/ui/hooks/useToolScheduler.test.ts
View File

@ -48,6 +48,7 @@ const mockToolRegistry = {
const mockConfig = {
getToolRegistry: vi.fn(() => mockToolRegistry as unknown as ToolRegistry),
getApprovalMode: vi.fn(() => ApprovalMode.DEFAULT),
getDisableDataCollection: () => false,
};
const mockTool: Tool = {

17
packages/core/src/config/config.ts
View File

@ -33,8 +33,10 @@ import {
DEFAULT_TELEMETRY_TARGET,
DEFAULT_OTLP_ENDPOINT,
TelemetryTarget,
StartSessionEvent,
} from '../telemetry/index.js';
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from './models.js';
import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js';
export enum ApprovalMode {
DEFAULT = 'default',
@ -55,6 +57,7 @@ export interface TelemetrySettings {
target?: TelemetryTarget;
otlpEndpoint?: string;
logPrompts?: boolean;
disableDataCollection?: boolean;
}
export class MCPServerConfig {
@ -114,6 +117,7 @@ export interface ConfigParameters {
fileDiscoveryService?: FileDiscoveryService;
bugCommand?: BugCommandSettings;
model: string;
disableDataCollection?: boolean;
}
export class Config {
@ -150,6 +154,7 @@ export class Config {
private readonly cwd: string;
private readonly bugCommand: BugCommandSettings | undefined;
private readonly model: string;
private readonly disableDataCollection: boolean;
constructor(params: ConfigParameters) {
this.sessionId = params.sessionId;
@ -189,6 +194,8 @@ export class Config {
this.fileDiscoveryService = params.fileDiscoveryService ?? null;
this.bugCommand = params.bugCommand;
this.model = params.model;
this.disableDataCollection =
params.telemetry?.disableDataCollection ?? true;
if (params.contextFileName) {
setGeminiMdFilename(params.contextFileName);
@ -197,6 +204,12 @@ export class Config {
if (this.telemetrySettings.enabled) {
initializeTelemetry(this);
}
if (!this.disableDataCollection) {
ClearcutLogger.getInstance(this)?.enqueueLogEvent(
new StartSessionEvent(this),
);
}
}
async refreshAuth(authMethod: AuthType) {
@ -370,6 +383,10 @@ export class Config {
return this.fileDiscoveryService;
}
getDisableDataCollection(): boolean {
return this.disableDataCollection;
}
async getGitService(): Promise<GitService> {
if (!this.gitService) {
this.gitService = new GitService(this.targetDir);

1
packages/core/src/core/coreToolScheduler.test.ts
View File

@ -77,6 +77,7 @@ describe('CoreToolScheduler', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getDisableDataCollection: () => false,
} as Config;
const scheduler = new CoreToolScheduler({

16
packages/core/src/core/coreToolScheduler.ts
View File

@ -16,6 +16,7 @@ import {
EditorType,
Config,
logToolCall,
ToolCallEvent,
} from '../index.js';
import { Part, PartListUnion } from '@google/genai';
import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js';
@ -652,20 +653,7 @@ export class CoreToolScheduler {
this.toolCalls = [];
for (const call of completedCalls) {
logToolCall(
this.config,
{
function_name: call.request.name,
function_args: call.request.args,
duration_ms: call.durationMs ?? 0,
success: call.status === 'success',
error:
call.status === 'error'
? call.response.error?.message
: undefined,
},
call.outcome,
);
logToolCall(this.config, new ToolCallEvent(call));
}
if (this.onAllToolCallsComplete) {

1
packages/core/src/core/geminiChat.test.ts
View File

@ -27,6 +27,7 @@ const mockModelsModule = {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryLogPromptsEnabled: () => true,
getDisableDataCollection: () => false,
} as unknown as Config;
describe('GeminiChat', () => {

54
packages/core/src/core/geminiChat.ts
View File

@ -24,12 +24,16 @@ import {
logApiRequest,
logApiResponse,
logApiError,
getFinalUsageMetadata,
} from '../telemetry/loggers.js';
import {
getStructuredResponse,
getStructuredResponseFromParts,
} from '../utils/generateContentResponseUtilities.js';
import {
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
} from '../telemetry/types.js';
/**
* Returns true if the response is valid, false otherwise.
@ -152,14 +156,8 @@ export class GeminiChat {
contents: Content[],
model: string,
): Promise<void> {
const shouldLogUserPrompts = (config: Config): boolean =>
config.getTelemetryLogPromptsEnabled() ?? false;
const requestText = this._getRequestTextFromContents(contents);
logApiRequest(this.config, {
model,
request_text: shouldLogUserPrompts(this.config) ? requestText : undefined,
});
logApiRequest(this.config, new ApiRequestEvent(model, requestText));
}
private async _logApiResponse(
@ -167,31 +165,20 @@ export class GeminiChat {
usageMetadata?: GenerateContentResponseUsageMetadata,
responseText?: string,
): Promise<void> {
logApiResponse(this.config, {
model: this.model,
duration_ms: durationMs,
status_code: 200, // Assuming 200 for success
input_token_count: usageMetadata?.promptTokenCount ?? 0,
output_token_count: usageMetadata?.candidatesTokenCount ?? 0,
cached_content_token_count: usageMetadata?.cachedContentTokenCount ?? 0,
thoughts_token_count: usageMetadata?.thoughtsTokenCount ?? 0,
tool_token_count: usageMetadata?.toolUsePromptTokenCount ?? 0,
response_text: responseText,
});
logApiResponse(
this.config,
new ApiResponseEvent(this.model, durationMs, usageMetadata, responseText),
);
}
private _logApiError(durationMs: number, error: unknown): void {
const errorMessage = error instanceof Error ? error.message : String(error);
const errorType = error instanceof Error ? error.name : 'unknown';
const statusCode = 'unknown';
logApiError(this.config, {
model: this.model,
error: errorMessage,
status_code: statusCode,
error_type: errorType,
duration_ms: durationMs,
});
logApiError(
this.config,
new ApiErrorEvent(this.model, errorMessage, durationMs, errorType),
);
}
/**
@ -402,6 +389,17 @@ export class GeminiChat {
this.history = history;
}
getFinalUsageMetadata(
chunks: GenerateContentResponse[],
): GenerateContentResponseUsageMetadata | undefined {
const lastChunkWithMetadata = chunks
.slice()
.reverse()
.find((chunk) => chunk.usageMetadata);
return lastChunkWithMetadata?.usageMetadata;
}
private async *processStreamResponse(
streamResponse: AsyncGenerator<GenerateContentResponse>,
inputContent: Content,
@ -444,7 +442,7 @@ export class GeminiChat {
const fullText = getStructuredResponseFromParts(allParts);
await this._logApiResponse(
durationMs,
getFinalUsageMetadata(chunks),
this.getFinalUsageMetadata(chunks),
fullText,
);
}

5
packages/core/src/core/nonInteractiveToolExecutor.test.ts
View File

@ -16,7 +16,10 @@ import {
} from '../index.js';
import { Part, Type } from '@google/genai';
const mockConfig = {} as unknown as Config;
const mockConfig = {
getSessionId: () => 'test-session-id',
getDisableDataCollection: () => false,
} as unknown as Config;
describe('executeToolCall', () => {
let mockToolRegistry: ToolRegistry;

6
packages/core/src/core/nonInteractiveToolExecutor.ts
View File

@ -33,6 +33,8 @@ export async function executeToolCall(
);
const durationMs = Date.now() - startTime;
logToolCall(config, {
'event.name': 'tool_call',
'event.timestamp': new Date().toISOString(),
function_name: toolCallRequest.name,
function_args: toolCallRequest.args,
duration_ms: durationMs,
@ -67,6 +69,8 @@ export async function executeToolCall(
const durationMs = Date.now() - startTime;
logToolCall(config, {
'event.name': 'tool_call',
'event.timestamp': new Date().toISOString(),
function_name: toolCallRequest.name,
function_args: toolCallRequest.args,
duration_ms: durationMs,
@ -89,6 +93,8 @@ export async function executeToolCall(
const error = e instanceof Error ? e : new Error(String(e));
const durationMs = Date.now() - startTime;
logToolCall(config, {
'event.name': 'tool_call',
'event.timestamp': new Date().toISOString(),
function_name: toolCallRequest.name,
function_args: toolCallRequest.args,
duration_ms: durationMs,

338
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts Normal file
View File

@ -0,0 +1,338 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Buffer } from 'buffer';
import * as https from 'https';
import {
StartSessionEvent,
EndSessionEvent,
UserPromptEvent,
ToolCallEvent,
ApiRequestEvent,
ApiResponseEvent,
ApiErrorEvent,
} from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js';
import { Config } from '../../config/config.js';
import { getPersistentUserId } from '../../utils/user_id.js';
const start_session_event_name = 'start_session';
const new_prompt_event_name = 'new_prompt';
const tool_call_event_name = 'tool_call';
const api_request_event_name = 'api_request';
const api_response_event_name = 'api_response';
const api_error_event_name = 'api_error';
const end_session_event_name = 'end_session';
export interface LogResponse {
nextRequestWaitMs?: number;
}
// Singleton class for batch posting log events to Clearcut. When a new event comes in, the elapsed time
// is checked and events are flushed to Clearcut if at least a minute has passed since the last flush.
export class ClearcutLogger {
private static instance: ClearcutLogger;
private config?: Config;
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
private readonly events: any = [];
private last_flush_time: number = Date.now();
private flush_interval_ms: number = 1000 * 60; // Wait at least a minute before flushing events.
private constructor(config?: Config) {
this.config = config;
}
static getInstance(config?: Config): ClearcutLogger | undefined {
if (config === undefined || config?.getDisableDataCollection())
return undefined;
if (!ClearcutLogger.instance) {
ClearcutLogger.instance = new ClearcutLogger(config);
}
return ClearcutLogger.instance;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
enqueueLogEvent(event: any): void {
this.events.push([
{
event_time_ms: Date.now(),
source_extension_json: JSON.stringify(event),
},
]);
}
createLogEvent(name: string, data: Map<EventMetadataKey, string>): object {
return {
Application: 'GEMINI_CLI',
event_name: name,
client_install_id: getPersistentUserId(),
event_metadata: [data] as object[],
};
}
flushIfNeeded(): void {
if (Date.now() - this.last_flush_time < this.flush_interval_ms) {
return;
}
this.flushToClearcut();
this.last_flush_time = Date.now();
}
flushToClearcut(): Promise<LogResponse> {
return new Promise<Buffer>((resolve, reject) => {
const request = [
{
log_source_name: 'CONCORD',
request_time_ms: Date.now(),
log_event: this.events,
},
];
const body = JSON.stringify(request);
const options = {
hostname: 'play.googleapis.com',
path: '/log',
method: 'POST',
headers: { 'Content-Length': Buffer.byteLength(body) },
};
const bufs: Buffer[] = [];
const req = https.request(options, (res) => {
res.on('data', (buf) => bufs.push(buf));
res.on('end', () => {
resolve(Buffer.concat(bufs));
});
});
req.on('error', (e) => {
reject(e);
});
req.end(body);
}).then((buf: Buffer) => {
try {
this.events.length = 0;
return this.decodeLogResponse(buf) || {};
} catch (error: unknown) {
console.error('Error flushing log events:', error);
return {};
}
});
}
// Visible for testing. Decodes protobuf-encoded response from Clearcut server.
decodeLogResponse(buf: Buffer): LogResponse | undefined {
// TODO(obrienowen): return specific errors to facilitate debugging.
if (buf.length < 1) {
return undefined;
}
// The first byte of the buffer is `field<<3 | type`. We're looking for field
// 1, with type varint, represented by type=0. If the first byte isn't 8, that
// means field 1 is missing or the message is corrupted. Either way, we return
// undefined.
if (buf.readUInt8(0) !== 8) {
return undefined;
}
let ms = BigInt(0);
let cont = true;
// In each byte, the most significant bit is the continuation bit. If it's
// set, we keep going. The lowest 7 bits, are data bits. They are concatenated
// in reverse order to form the final number.
for (let i = 1; cont && i < buf.length; i++) {
const byte = buf.readUInt8(i);
ms |= BigInt(byte & 0x7f) << BigInt(7 * (i - 1));
cont = (byte & 0x80) !== 0;
}
if (cont) {
// We have fallen off the buffer without seeing a terminating byte. The
// message is corrupted.
return undefined;
}
return {
nextRequestWaitMs: Number(ms),
};
}
logStartSessionEvent(event: StartSessionEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(EventMetadataKey.GEMINI_CLI_START_SESSION_MODEL, event.model);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_EMBEDDING_MODEL,
event.embedding_model,
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_SANDBOX,
event.sandbox_enabled.toString(),
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_CORE_TOOLS,
event.core_tools_enabled,
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_APPROVAL_MODE,
event.approval_mode,
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_API_KEY_ENABLED,
event.api_key_enabled.toString(),
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED,
event.vertex_ai_enabled.toString(),
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED,
event.debug_enabled.toString(),
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_MCP_SERVERS,
event.mcp_servers,
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED,
event.telemetry_enabled.toString(),
);
data.set(
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED,
event.telemetry_log_user_prompts_enabled.toString(),
);
this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data));
this.flushIfNeeded();
}
logNewPromptEvent(event: UserPromptEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(
EventMetadataKey.GEMINI_CLI_USER_PROMPT_LENGTH,
JSON.stringify(event.prompt_length),
);
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
this.flushIfNeeded();
}
logToolCallEvent(event: ToolCallEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(EventMetadataKey.GEMINI_CLI_TOOL_CALL_NAME, event.function_name);
data.set(
EventMetadataKey.GEMINI_CLI_TOOL_CALL_DECISION,
JSON.stringify(event.decision),
);
data.set(
EventMetadataKey.GEMINI_CLI_TOOL_CALL_SUCCESS,
JSON.stringify(event.success),
);
data.set(
EventMetadataKey.GEMINI_CLI_TOOL_CALL_DURATION_MS,
JSON.stringify(event.duration_ms),
);
data.set(
EventMetadataKey.GEMINI_CLI_TOOL_ERROR_MESSAGE,
JSON.stringify(event.error),
);
data.set(
EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
JSON.stringify(event.error_type),
);
this.enqueueLogEvent(this.createLogEvent(tool_call_event_name, data));
this.flushIfNeeded();
}
logApiRequestEvent(event: ApiRequestEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(EventMetadataKey.GEMINI_CLI_API_REQUEST_MODEL, event.model);
this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data));
this.flushIfNeeded();
}
logApiResponseEvent(event: ApiResponseEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(EventMetadataKey.GEMINI_CLI_API_RESPONSE_MODEL, event.model);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_STATUS_CODE,
JSON.stringify(event.status_code),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_DURATION_MS,
JSON.stringify(event.duration_ms),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_ERROR_MESSAGE,
JSON.stringify(event.error),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT,
JSON.stringify(event.input_token_count),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT,
JSON.stringify(event.output_token_count),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT,
JSON.stringify(event.cached_content_token_count),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT,
JSON.stringify(event.thoughts_token_count),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
JSON.stringify(event.tool_token_count),
);
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
this.flushIfNeeded();
}
logApiErrorEvent(event: ApiErrorEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(EventMetadataKey.GEMINI_CLI_API_ERROR_MODEL, event.model);
data.set(
EventMetadataKey.GEMINI_CLI_API_ERROR_TYPE,
JSON.stringify(event.error_type),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_ERROR_STATUS_CODE,
JSON.stringify(event.status_code),
);
data.set(
EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS,
JSON.stringify(event.duration_ms),
);
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
this.flushIfNeeded();
}
logEndSessionEvent(event: EndSessionEvent): void {
const data: Map<EventMetadataKey, string> = new Map();
data.set(
EventMetadataKey.GEMINI_CLI_END_SESSION_ID,
event?.session_id?.toString() ?? '',
);
this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data));
// Flush immediately on session end.
this.flushToClearcut();
}
shutdown() {
const event = new EndSessionEvent(this.config);
this.logEndSessionEvent(event);
}
}

153
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts Normal file
View File

@ -0,0 +1,153 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
// Defines valid event metadata keys for Clearcut logging.
export enum EventMetadataKey {
GEMINI_CLI_KEY_UNKNOWN = 0,
// ==========================================================================
// Start Session Event Keys
// ===========================================================================
// Logs the model id used in the session.
GEMINI_CLI_START_SESSION_MODEL = 1,
// Logs the embedding model id used in the session.
GEMINI_CLI_START_SESSION_EMBEDDING_MODEL = 2,
// Logs the sandbox that was used in the session.
GEMINI_CLI_START_SESSION_SANDBOX = 3,
// Logs the core tools that were enabled in the session.
GEMINI_CLI_START_SESSION_CORE_TOOLS = 4,
// Logs the approval mode that was used in the session.
GEMINI_CLI_START_SESSION_APPROVAL_MODE = 5,
// Logs whether an API key was used in the session.
GEMINI_CLI_START_SESSION_API_KEY_ENABLED = 6,
// Logs whether the Vertex API was used in the session.
GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED = 7,
// Logs whether debug mode was enabled in the session.
GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED = 8,
// Logs the MCP servers that were enabled in the session.
GEMINI_CLI_START_SESSION_MCP_SERVERS = 9,
// Logs whether user-collected telemetry was enabled in the session.
GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED = 10,
// Logs whether prompt collection was enabled for user-collected telemetry.
GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED = 11,
// Logs whether the session was configured to respect gitignore files.
GEMINI_CLI_START_SESSION_RESPECT_GITIGNORE = 12,
// ==========================================================================
// User Prompt Event Keys
// ===========================================================================
// Logs the length of the prompt.
GEMINI_CLI_USER_PROMPT_LENGTH = 13,
// ==========================================================================
// Tool Call Event Keys
// ===========================================================================
// Logs the function name.
GEMINI_CLI_TOOL_CALL_NAME = 14,
// Logs the user's decision about how to handle the tool call.
GEMINI_CLI_TOOL_CALL_DECISION = 15,
// Logs whether the tool call succeeded.
GEMINI_CLI_TOOL_CALL_SUCCESS = 16,
// Logs the tool call duration in milliseconds.
GEMINI_CLI_TOOL_CALL_DURATION_MS = 17,
// Logs the tool call error message, if any.
GEMINI_CLI_TOOL_ERROR_MESSAGE = 18,
// Logs the tool call error type, if any.
GEMINI_CLI_TOOL_CALL_ERROR_TYPE = 19,
// ==========================================================================
// GenAI API Request Event Keys
// ===========================================================================
// Logs the model id of the request.
GEMINI_CLI_API_REQUEST_MODEL = 20,
// ==========================================================================
// GenAI API Response Event Keys
// ===========================================================================
// Logs the model id of the API call.
GEMINI_CLI_API_RESPONSE_MODEL = 21,
// Logs the status code of the response.
GEMINI_CLI_API_RESPONSE_STATUS_CODE = 22,
// Logs the duration of the API call in milliseconds.
GEMINI_CLI_API_RESPONSE_DURATION_MS = 23,
// Logs the error message of the API call, if any.
GEMINI_CLI_API_ERROR_MESSAGE = 24,
// Logs the input token count of the API call.
GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT = 25,
// Logs the output token count of the API call.
GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT = 26,
// Logs the cached token count of the API call.
GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT = 27,
// Logs the thinking token count of the API call.
GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT = 28,
// Logs the tool use token count of the API call.
GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT = 29,
// ==========================================================================
// GenAI API Error Event Keys
// ===========================================================================
// Logs the model id of the API call.
GEMINI_CLI_API_ERROR_MODEL = 30,
// Logs the error type.
GEMINI_CLI_API_ERROR_TYPE = 31,
// Logs the status code of the error response.
GEMINI_CLI_API_ERROR_STATUS_CODE = 32,
// Logs the duration of the API call in milliseconds.
GEMINI_CLI_API_ERROR_DURATION_MS = 33,
// ==========================================================================
// End Session Event Keys
// ===========================================================================
// Logs the end of a session.
GEMINI_CLI_END_SESSION_ID = 34,
}
export function getEventMetadataKey(
keyName: string,
): EventMetadataKey | undefined {
// Access the enum member by its string name
const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey];
// Check if the result is a valid enum member (not undefined and is a number)
if (typeof key === 'number') {
return key;
}
return undefined;
}

4
packages/core/src/telemetry/index.ts
View File

@ -25,15 +25,15 @@ export {
logApiRequest,
logApiError,
logApiResponse,
getFinalUsageMetadata,
} from './loggers.js';
export {
StartSessionEvent,
EndSessionEvent,
UserPromptEvent,
ToolCallEvent,
ApiRequestEvent,
ApiErrorEvent,
ApiResponseEvent,
CliConfigEvent,
TelemetryEvent,
} from './types.js';
export { SpanStatusCode, ValueType } from '@opentelemetry/api';

377
packages/core/src/telemetry/loggers.test.ts
View File

@ -4,14 +4,24 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { ToolConfirmationOutcome } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js';
import {
AuthType,
CompletedToolCall,
ContentGeneratorConfig,
EditTool,
ErroredToolCall,
GeminiClient,
ToolConfirmationOutcome,
ToolRegistry,
} from '../index.js';
import { logs } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
import {
EVENT_API_REQUEST,
EVENT_API_RESPONSE,
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
} from './constants.js';
import {
@ -20,13 +30,19 @@ import {
logCliConfiguration,
logUserPrompt,
logToolCall,
ToolCallDecision,
getFinalUsageMetadata,
} from './loggers.js';
import {
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
ToolCallDecision,
ToolCallEvent,
UserPromptEvent,
} from './types.js';
import * as metrics from './metrics.js';
import * as sdk from './sdk.js';
import { vi, describe, beforeEach, it, expect } from 'vitest';
import { GenerateContentResponse } from '@google/genai';
import { GenerateContentResponseUsageMetadata } from '@google/genai';
describe('loggers', () => {
const mockLogger = {
@ -54,8 +70,11 @@ describe('loggers', () => {
apiKey: 'test-api-key',
authType: AuthType.USE_VERTEX_AI,
}),
getTelemetryEnabled: () => true,
getDisableDataCollection: () => false,
getTelemetryLogPromptsEnabled: () => true,
getFileFilteringRespectGitIgnore: () => true,
getFileFilteringAllowBuildArtifacts: () => false,
getDebugMode: () => true,
getMcpServers: () => ({
'test-server': {
@ -63,15 +82,18 @@ describe('loggers', () => {
},
}),
getQuestion: () => 'test-question',
getTargetDir: () => 'target-dir',
getProxy: () => 'http://test.proxy.com:8080',
} as unknown as Config;
logCliConfiguration(mockConfig);
const startSessionEvent = new StartSessionEvent(mockConfig);
logCliConfiguration(mockConfig, startSessionEvent);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'CLI configuration loaded.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.config',
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': '2025-01-01T00:00:00.000Z',
model: 'test-model',
embedding_model: 'test-embedding-model',
@ -92,14 +114,13 @@ describe('loggers', () => {
describe('logUserPrompt', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
getDisableDataCollection: () => false,
} as unknown as Config;
it('should log a user prompt', () => {
const event = {
prompt: 'test-prompt',
prompt_length: 11,
};
const event = new UserPromptEvent(11, 'test-prompt');
logUserPrompt(mockConfig, event);
@ -118,12 +139,12 @@ describe('loggers', () => {
it('should not log prompt if disabled', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => false,
getTargetDir: () => 'target-dir',
getDisableDataCollection: () => false,
} as unknown as Config;
const event = {
prompt: 'test-prompt',
prompt_length: 11,
};
const event = new UserPromptEvent(11, 'test-prompt');
logUserPrompt(mockConfig, event);
@ -142,6 +163,10 @@ describe('loggers', () => {
describe('logApiResponse', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getDisableDataCollection: () => false,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
const mockMetrics = {
@ -159,17 +184,19 @@ describe('loggers', () => {
});
it('should log an API response with all fields', () => {
const event = {
model: 'test-model',
status_code: 200,
duration_ms: 100,
input_token_count: 17,
output_token_count: 50,
cached_content_token_count: 10,
thoughts_token_count: 5,
tool_token_count: 2,
response_text: 'test-response',
const usageData: GenerateContentResponseUsageMetadata = {
promptTokenCount: 17,
candidatesTokenCount: 50,
cachedContentTokenCount: 10,
thoughtsTokenCount: 5,
toolUsePromptTokenCount: 2,
};
const event = new ApiResponseEvent(
'test-model',
100,
usageData,
'test-response',
);
logApiResponse(mockConfig, event);
@ -209,22 +236,25 @@ describe('loggers', () => {
});
it('should log an API response with an error', () => {
const event = {
model: 'test-model',
duration_ms: 100,
error: 'test-error',
input_token_count: 17,
output_token_count: 50,
cached_content_token_count: 10,
thoughts_token_count: 5,
tool_token_count: 2,
response_text: 'test-response',
const usageData: GenerateContentResponseUsageMetadata = {
promptTokenCount: 17,
candidatesTokenCount: 50,
cachedContentTokenCount: 10,
thoughtsTokenCount: 5,
toolUsePromptTokenCount: 2,
};
const event = new ApiResponseEvent(
'test-model',
100,
usageData,
'test-response',
'test-error',
);
logApiResponse(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API response from test-model. Status: N/A. Duration: 100ms.',
body: 'API response from test-model. Status: 200. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
...event,
@ -239,13 +269,14 @@ describe('loggers', () => {
describe('logApiRequest', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getDisableDataCollection: () => false,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
it('should log an API request with request_text', () => {
const event = {
model: 'test-model',
request_text: 'This is a test request',
};
const event = new ApiRequestEvent('test-model', 'This is a test request');
logApiRequest(mockConfig, event);
@ -262,9 +293,7 @@ describe('loggers', () => {
});
it('should log an API request without request_text', () => {
const event = {
model: 'test-model',
};
const event = new ApiRequestEvent('test-model');
logApiRequest(mockConfig, event);
@ -281,8 +310,46 @@ describe('loggers', () => {
});
describe('logToolCall', () => {
const cfg1 = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getGeminiClient: () => mockGeminiClient,
} as Config;
const cfg2 = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getProxy: () => 'http://test.proxy.com:8080',
getContentGeneratorConfig: () =>
({ model: 'test-model' }) as ContentGeneratorConfig,
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding-model',
getWorkingDir: () => 'test-working-dir',
getSandbox: () => true,
getCoreTools: () => ['ls', 'read-file'],
getApprovalMode: () => 'default',
getTelemetryLogPromptsEnabled: () => true,
getFileFilteringRespectGitIgnore: () => true,
getFileFilteringAllowBuildArtifacts: () => false,
getDebugMode: () => true,
getMcpServers: () => ({
'test-server': {
command: 'test-command',
},
}),
getQuestion: () => 'test-question',
getToolRegistry: () => new ToolRegistry(cfg1),
getFullContext: () => false,
getUserMemory: () => 'user-memory',
} as unknown as Config;
const mockGeminiClient = new GeminiClient(cfg2);
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getGeminiClient: () => mockGeminiClient,
getDisableDataCollection: () => false,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
const mockMetrics = {
@ -297,23 +364,36 @@ describe('loggers', () => {
});
it('should log a tool call with all fields', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
},
duration_ms: 100,
success: true,
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
outcome: ToolConfirmationOutcome.ProceedOnce,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event, ToolConfirmationOutcome.ProceedOnce);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
@ -339,23 +419,35 @@ describe('loggers', () => {
);
});
it('should log a tool call with a reject decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
const call: ErroredToolCall = {
status: 'error',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
},
duration_ms: 100,
success: false,
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
durationMs: 100,
outcome: ToolConfirmationOutcome.Cancel,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event, ToolConfirmationOutcome.Cancel);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
@ -382,23 +474,36 @@ describe('loggers', () => {
});
it('should log a tool call with a modify decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
},
duration_ms: 100,
success: true,
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
outcome: ToolConfirmationOutcome.ModifyWithEditor,
tool: new EditTool(mockConfig),
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event, ToolConfirmationOutcome.ModifyWithEditor);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
@ -425,15 +530,27 @@ describe('loggers', () => {
});
it('should log a tool call without a decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
},
duration_ms: 100,
success: true,
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
@ -441,7 +558,7 @@ describe('loggers', () => {
body: 'Tool call: test-function. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
@ -467,17 +584,29 @@ describe('loggers', () => {
});
it('should log a failed tool call with an error', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
const call: ErroredToolCall = {
status: 'error',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
},
duration_ms: 100,
success: false,
error: 'test-error',
error_type: 'test-error-type',
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: {
name: 'test-error-type',
message: 'test-error',
},
},
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
@ -485,7 +614,7 @@ describe('loggers', () => {
body: 'Tool call: test-function. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
@ -515,75 +644,3 @@ describe('loggers', () => {
});
});
});
describe('getFinalUsageMetadata', () => {
const createMockResponse = (
usageMetadata?: GenerateContentResponse['usageMetadata'],
): GenerateContentResponse =>
({
text: () => '',
data: () => ({}) as Record<string, unknown>,
functionCalls: () => [],
executableCode: () => [],
codeExecutionResult: () => [],
usageMetadata,
}) as unknown as GenerateContentResponse;
it('should return the usageMetadata from the last chunk that has it', () => {
const chunks: GenerateContentResponse[] = [
createMockResponse({
promptTokenCount: 10,
candidatesTokenCount: 20,
totalTokenCount: 30,
}),
createMockResponse(),
createMockResponse({
promptTokenCount: 15,
candidatesTokenCount: 25,
totalTokenCount: 40,
}),
createMockResponse(),
];
const result = getFinalUsageMetadata(chunks);
expect(result).toEqual({
promptTokenCount: 15,
candidatesTokenCount: 25,
totalTokenCount: 40,
});
});
it('should return undefined if no chunks have usageMetadata', () => {
const chunks: GenerateContentResponse[] = [
createMockResponse(),
createMockResponse(),
createMockResponse(),
];
const result = getFinalUsageMetadata(chunks);
expect(result).toBeUndefined();
});
it('should return the metadata from the only chunk if it has it', () => {
const chunks: GenerateContentResponse[] = [
createMockResponse({
promptTokenCount: 1,
candidatesTokenCount: 2,
totalTokenCount: 3,
}),
];
const result = getFinalUsageMetadata(chunks);
expect(result).toEqual({
promptTokenCount: 1,
candidatesTokenCount: 2,
totalTokenCount: 3,
});
});
it('should return undefined for an empty array of chunks', () => {
const chunks: GenerateContentResponse[] = [];
const result = getFinalUsageMetadata(chunks);
expect(result).toBeUndefined();
});
});

131
packages/core/src/telemetry/loggers.ts
View File

@ -20,6 +20,7 @@ import {
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
ToolCallEvent,
UserPromptEvent,
} from './types.js';
@ -30,15 +31,10 @@ import {
recordToolCallMetrics,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import { ToolConfirmationOutcome } from '../tools/tools.js';
import {
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
} from '@google/genai';
import { AuthType } from '../core/contentGenerator.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
const shouldLogUserPrompts = (config: Config): boolean =>
config.getTelemetryLogPromptsEnabled() ?? false;
config.getTelemetryLogPromptsEnabled();
function getCommonAttributes(config: Config): LogAttributes {
return {
@ -46,59 +42,30 @@ function getCommonAttributes(config: Config): LogAttributes {
};
}
export enum ToolCallDecision {
ACCEPT = 'accept',
REJECT = 'reject',
MODIFY = 'modify',
}
export function getDecisionFromOutcome(
outcome: ToolConfirmationOutcome,
): ToolCallDecision {
switch (outcome) {
case ToolConfirmationOutcome.ProceedOnce:
case ToolConfirmationOutcome.ProceedAlways:
case ToolConfirmationOutcome.ProceedAlwaysServer:
case ToolConfirmationOutcome.ProceedAlwaysTool:
return ToolCallDecision.ACCEPT;
case ToolConfirmationOutcome.ModifyWithEditor:
return ToolCallDecision.MODIFY;
case ToolConfirmationOutcome.Cancel:
default:
return ToolCallDecision.REJECT;
}
}
export function logCliConfiguration(config: Config): void {
export function logCliConfiguration(
config: Config,
event: StartSessionEvent,
): void {
ClearcutLogger.getInstance(config)?.logStartSessionEvent(event);
if (!isTelemetrySdkInitialized()) return;
const generatorConfig = config.getContentGeneratorConfig();
let useGemini = false;
let useVertex = false;
if (generatorConfig && generatorConfig.authType) {
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
}
const mcpServers = config.getMcpServers();
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': new Date().toISOString(),
model: config.getModel(),
embedding_model: config.getEmbeddingModel(),
sandbox_enabled: !!config.getSandbox(),
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
approval_mode: config.getApprovalMode(),
api_key_enabled: useGemini || useVertex,
vertex_ai_enabled: useVertex,
log_user_prompts_enabled: config.getTelemetryLogPromptsEnabled(),
file_filtering_respect_git_ignore:
config.getFileFilteringRespectGitIgnore(),
debug_mode: config.getDebugMode(),
mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '',
model: event.model,
embedding_model: event.embedding_model,
sandbox_enabled: event.sandbox_enabled,
core_tools_enabled: event.core_tools_enabled,
approval_mode: event.approval_mode,
api_key_enabled: event.api_key_enabled,
vertex_ai_enabled: event.vertex_ai_enabled,
log_user_prompts_enabled: event.telemetry_log_user_prompts_enabled,
file_filtering_respect_git_ignore: event.file_filtering_respect_git_ignore,
debug_mode: event.debug_enabled,
mcp_servers: event.mcp_servers,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: 'CLI configuration loaded.',
@ -107,12 +74,8 @@ export function logCliConfiguration(config: Config): void {
logger.emit(logRecord);
}
export function logUserPrompt(
config: Config,
event: Omit<UserPromptEvent, 'event.name' | 'event.timestamp' | 'prompt'> & {
prompt: string;
},
): void {
export function logUserPrompt(config: Config, event: UserPromptEvent): void {
ClearcutLogger.getInstance(config)?.logNewPromptEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
@ -134,22 +97,16 @@ export function logUserPrompt(
logger.emit(logRecord);
}
export function logToolCall(
config: Config,
event: Omit<ToolCallEvent, 'event.name' | 'event.timestamp' | 'decision'>,
outcome?: ToolConfirmationOutcome,
): void {
export function logToolCall(config: Config, event: ToolCallEvent): void {
ClearcutLogger.getInstance(config)?.logToolCallEvent(event);
if (!isTelemetrySdkInitialized()) return;
const decision = outcome ? getDecisionFromOutcome(outcome) : undefined;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': new Date().toISOString(),
function_args: JSON.stringify(event.function_args, null, 2),
decision,
};
if (event.error) {
attributes['error.message'] = event.error;
@ -157,9 +114,10 @@ export function logToolCall(
attributes['error.type'] = event.error_type;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Tool call: ${event.function_name}${decision ? `. Decision: ${decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
body: `Tool call: ${event.function_name}${event.decision ? `. Decision: ${event.decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
@ -168,21 +126,21 @@ export function logToolCall(
event.function_name,
event.duration_ms,
event.success,
decision,
event.decision,
);
}
export function logApiRequest(
config: Config,
event: Omit<ApiRequestEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiRequest(config: Config, event: ApiRequestEvent): void {
ClearcutLogger.getInstance(config)?.logApiRequestEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_REQUEST,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API request to ${event.model}.`,
@ -191,17 +149,18 @@ export function logApiRequest(
logger.emit(logRecord);
}
export function logApiError(
config: Config,
event: Omit<ApiErrorEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiError(config: Config, event: ApiErrorEvent): void {
ClearcutLogger.getInstance(config)?.logApiErrorEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': new Date().toISOString(),
['error.message']: event.error,
model_name: event.model,
duration: event.duration_ms,
};
if (event.error_type) {
@ -226,10 +185,8 @@ export function logApiError(
);
}
export function logApiResponse(
config: Config,
event: Omit<ApiResponseEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
ClearcutLogger.getInstance(config)?.logApiResponseEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
@ -287,15 +244,3 @@ export function logApiResponse(
);
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
}
export function getFinalUsageMetadata(
chunks: GenerateContentResponse[],
): GenerateContentResponseUsageMetadata | undefined {
// Only the last streamed item has the final token count.
const lastChunkWithMetadata = chunks
.slice()
.reverse()
.find((chunk) => chunk.usageMetadata);
return lastChunkWithMetadata?.usageMetadata;
}

5
packages/core/src/telemetry/sdk.ts
View File

@ -29,6 +29,8 @@ import { Config } from '../config/config.js';
import { SERVICE_NAME } from './constants.js';
import { initializeMetrics } from './metrics.js';
import { logCliConfiguration } from './loggers.js';
import { StartSessionEvent } from './types.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
// For troubleshooting, set the log level to DiagLogLevel.DEBUG
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
@ -113,7 +115,7 @@ export function initializeTelemetry(config: Config): void {
console.log('OpenTelemetry SDK started successfully.');
telemetryInitialized = true;
initializeMetrics(config);
logCliConfiguration(config);
logCliConfiguration(config, new StartSessionEvent(config));
} catch (error) {
console.error('Error starting OpenTelemetry SDK:', error);
}
@ -127,6 +129,7 @@ export async function shutdownTelemetry(): Promise<void> {
return;
}
try {
ClearcutLogger.getInstance()?.shutdown();
await sdk.shutdown();
console.log('OpenTelemetry SDK shut down successfully.');
} catch (error) {

4
packages/core/src/telemetry/telemetry.test.ts
View File

@ -13,6 +13,7 @@ import {
import { Config } from '../config/config.js';
import { NodeSDK } from '@opentelemetry/sdk-node';
import * as loggers from './loggers.js';
import { StartSessionEvent } from './types.js';
vi.mock('@opentelemetry/sdk-node');
vi.mock('../config/config.js');
@ -55,10 +56,11 @@ describe('telemetry', () => {
it('should initialize the telemetry service', () => {
initializeTelemetry(mockConfig);
const event = new StartSessionEvent(mockConfig);
expect(NodeSDK).toHaveBeenCalled();
expect(mockNodeSdk.start).toHaveBeenCalled();
expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig);
expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig, event);
});
it('should shutdown the telemetry service', async () => {

177
packages/core/src/telemetry/types.ts
View File

@ -4,16 +4,108 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { ToolCallDecision } from './loggers.js';
import { GenerateContentResponseUsageMetadata } from '@google/genai';
import { Config } from '../config/config.js';
import { CompletedToolCall } from '../core/coreToolScheduler.js';
import { ToolConfirmationOutcome } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js';
export interface UserPromptEvent {
export enum ToolCallDecision {
ACCEPT = 'accept',
REJECT = 'reject',
MODIFY = 'modify',
}
export function getDecisionFromOutcome(
outcome: ToolConfirmationOutcome,
): ToolCallDecision {
switch (outcome) {
case ToolConfirmationOutcome.ProceedOnce:
case ToolConfirmationOutcome.ProceedAlways:
case ToolConfirmationOutcome.ProceedAlwaysServer:
case ToolConfirmationOutcome.ProceedAlwaysTool:
return ToolCallDecision.ACCEPT;
case ToolConfirmationOutcome.ModifyWithEditor:
return ToolCallDecision.MODIFY;
case ToolConfirmationOutcome.Cancel:
default:
return ToolCallDecision.REJECT;
}
}
export class StartSessionEvent {
'event.name': 'cli_config';
'event.timestamp': string; // ISO 8601
model: string;
embedding_model: string;
sandbox_enabled: boolean;
core_tools_enabled: string;
approval_mode: string;
api_key_enabled: boolean;
vertex_ai_enabled: boolean;
debug_enabled: boolean;
mcp_servers: string;
telemetry_enabled: boolean;
telemetry_log_user_prompts_enabled: boolean;
file_filtering_respect_git_ignore: boolean;
constructor(config: Config) {
const generatorConfig = config.getContentGeneratorConfig();
const mcpServers = config.getMcpServers();
let useGemini = false;
let useVertex = false;
if (generatorConfig && generatorConfig.authType) {
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
}
this['event.name'] = 'cli_config';
this.model = config.getModel();
this.embedding_model = config.getEmbeddingModel();
this.sandbox_enabled =
typeof config.getSandbox() === 'string' || !!config.getSandbox();
this.core_tools_enabled = (config.getCoreTools() ?? []).join(',');
this.approval_mode = config.getApprovalMode();
this.api_key_enabled = useGemini || useVertex;
this.vertex_ai_enabled = useVertex;
this.debug_enabled = config.getDebugMode();
this.mcp_servers = mcpServers ? Object.keys(mcpServers).join(',') : '';
this.telemetry_enabled = config.getTelemetryEnabled();
this.telemetry_log_user_prompts_enabled =
config.getTelemetryLogPromptsEnabled();
this.file_filtering_respect_git_ignore =
config.getFileFilteringRespectGitIgnore();
}
}
export class EndSessionEvent {
'event.name': 'end_session';
'event.timestamp': string; // ISO 8601
session_id?: string;
constructor(config?: Config) {
this['event.name'] = 'end_session';
this['event.timestamp'] = new Date().toISOString();
this.session_id = config?.getSessionId();
}
}
export class UserPromptEvent {
'event.name': 'user_prompt';
'event.timestamp': string; // ISO 8601
prompt_length: number;
prompt?: string;
constructor(prompt_length: number, prompt?: string) {
this['event.name'] = 'user_prompt';
this['event.timestamp'] = new Date().toISOString();
this.prompt_length = prompt_length;
this.prompt = prompt;
}
}
export interface ToolCallEvent {
export class ToolCallEvent {
'event.name': 'tool_call';
'event.timestamp': string; // ISO 8601
function_name: string;
@ -23,16 +115,37 @@ export interface ToolCallEvent {
decision?: ToolCallDecision;
error?: string;
error_type?: string;
constructor(call: CompletedToolCall) {
this['event.name'] = 'tool_call';
this['event.timestamp'] = new Date().toISOString();
this.function_name = call.request.name;
this.function_args = call.request.args;
this.duration_ms = call.durationMs ?? 0;
this.success = call.status === 'success';
this.decision = call.outcome
? getDecisionFromOutcome(call.outcome)
: undefined;
this.error = call.response.error?.message;
this.error_type = call.response.error?.name;
}
}
export interface ApiRequestEvent {
export class ApiRequestEvent {
'event.name': 'api_request';
'event.timestamp': string; // ISO 8601
model: string;
request_text?: string;
constructor(model: string, request_text?: string) {
this['event.name'] = 'api_request';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.request_text = request_text;
}
}
export interface ApiErrorEvent {
export class ApiErrorEvent {
'event.name': 'api_error';
'event.timestamp': string; // ISO 8601
model: string;
@ -40,9 +153,25 @@ export interface ApiErrorEvent {
error_type?: string;
status_code?: number | string;
duration_ms: number;
constructor(
model: string,
error: string,
duration_ms: number,
error_type?: string,
status_code?: number | string,
) {
this['event.name'] = 'api_error';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.error = error;
this.error_type = error_type;
this.status_code = status_code;
this.duration_ms = duration_ms;
}
}
export interface ApiResponseEvent {
export class ApiResponseEvent {
'event.name': 'api_response';
'event.timestamp': string; // ISO 8601
model: string;
@ -55,24 +184,34 @@ export interface ApiResponseEvent {
thoughts_token_count: number;
tool_token_count: number;
response_text?: string;
}
export interface CliConfigEvent {
'event.name': 'cli_config';
'event.timestamp': string; // ISO 8601
model: string;
sandbox_enabled: boolean;
core_tools_enabled: string;
approval_mode: string;
vertex_ai_enabled: boolean;
log_user_prompts_enabled: boolean;
file_filtering_respect_git_ignore: boolean;
constructor(
model: string,
duration_ms: number,
usage_data?: GenerateContentResponseUsageMetadata,
response_text?: string,
error?: string,
) {
this['event.name'] = 'api_response';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.duration_ms = duration_ms;
this.status_code = 200;
this.input_token_count = usage_data?.promptTokenCount ?? 0;
this.output_token_count = usage_data?.candidatesTokenCount ?? 0;
this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0;
this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0;
this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0;
this.response_text = response_text;
this.error = error;
}
}
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
| UserPromptEvent
| ToolCallEvent
| ApiRequestEvent
| ApiErrorEvent
| ApiResponseEvent
| CliConfigEvent;
| ApiResponseEvent;

58
packages/core/src/utils/user_id.ts Normal file
View File

@ -0,0 +1,58 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import * as os from 'os';
import * as fs from 'fs';
import * as path from 'path';
import { randomUUID } from 'crypto';
import { GEMINI_DIR } from './paths.js';
const homeDir = os.homedir() ?? '';
const geminiDir = path.join(homeDir, GEMINI_DIR);
const userIdFile = path.join(geminiDir, 'user_id');
function ensureGeminiDirExists() {
if (!fs.existsSync(geminiDir)) {
fs.mkdirSync(geminiDir, { recursive: true });
}
}
function readUserIdFromFile(): string | null {
if (fs.existsSync(userIdFile)) {
const userId = fs.readFileSync(userIdFile, 'utf-8').trim();
return userId || null;
}
return null;
}
function writeUserIdToFile(userId: string) {
fs.writeFileSync(userIdFile, userId, 'utf-8');
}
/**
* Retrieves the persistent user ID from a file, creating it if it doesn't exist.
* This ID is used for unique user tracking.
* @returns A UUID string for the user.
*/
export function getPersistentUserId(): string {
try {
ensureGeminiDirExists();
let userId = readUserIdFromFile();
if (!userId) {
userId = randomUUID();
writeUserIdToFile(userId);
}
return userId;
} catch (error) {
console.error(
'Error accessing persistent user ID file, generating ephemeral ID:',
error,
);
return '123456789';
}
}