From c0580eaf4b8b7f02a048ade43a0f0b652fa01129 Mon Sep 17 00:00:00 2001 From: jerop Date: Wed, 11 Jun 2025 20:15:44 +0000 Subject: [PATCH] feat(telemetry): expand cli configuration event Adds the following attributes to the event: - embedding_model - api_key_enabled - code_assist_enabled - debug_mode - mcp_servers This additional data will provide more insight into user configurations. --- docs/core/telemetry.md | 5 + packages/core/src/telemetry/loggers.test.ts | 214 +++++++++++++------- packages/core/src/telemetry/loggers.ts | 9 +- 3 files changed, 153 insertions(+), 75 deletions(-) diff --git a/docs/core/telemetry.md b/docs/core/telemetry.md index e7b82b65..695eacaf 100644 --- a/docs/core/telemetry.md +++ b/docs/core/telemetry.md @@ -262,13 +262,18 @@ These are timestamped records of specific events. - **Attributes**: - `model` (string) + - `embedding_model` (string) - `sandbox_enabled` (boolean) - `core_tools_enabled` (string) - `approval_mode` (string) + - `api_key_enabled` (boolean) - `vertex_ai_enabled` (boolean) + - `code_assist_enabled` (boolean) - `log_user_prompts_enabled` (boolean) - `file_filtering_respect_git_ignore` (boolean) - `file_filtering_allow_build_artifacts` (boolean) + - `debug_mode` (boolean) + - `mcp_servers` (string) - `gemini_cli.user_prompt`: Fired when a user submits a prompt. diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 3493dc49..283af47a 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -8,60 +8,105 @@ import { logs } from '@opentelemetry/api-logs'; import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import { Config } from '../config/config.js'; import { EVENT_API_RESPONSE } from './constants.js'; -import { logApiResponse } from './loggers.js'; +import { logApiResponse, logCliConfiguration } from './loggers.js'; import * as metrics from './metrics.js'; import * as sdk from './sdk.js'; import { vi, describe, beforeEach, it, expect } from 'vitest'; -describe('logApiResponse', () => { - const mockConfig = { - getSessionId: () => 'test-session-id', - } as Config; +vi.mock('@gemini-cli/cli/dist/src/utils/version', () => ({ + getCliVersion: () => 'test-version', +})); +vi.mock('@gemini-cli/cli/dist/src/config/settings', () => ({ + getTheme: () => 'test-theme', +})); + +describe('loggers', () => { const mockLogger = { emit: vi.fn(), }; - const mockMetrics = { - recordApiResponseMetrics: vi.fn(), - recordTokenUsageMetrics: vi.fn(), - }; - beforeEach(() => { vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(true); vi.spyOn(logs, 'getLogger').mockReturnValue(mockLogger); - vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation( - mockMetrics.recordApiResponseMetrics, - ); - vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation( - mockMetrics.recordTokenUsageMetrics, - ); vi.useFakeTimers(); vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z')); }); - it('should log an API response with all fields', () => { - const event = { - model: 'test-model', - status_code: 200, - duration_ms: 100, - attempt: 1, - output_token_count: 50, - cached_content_token_count: 10, - thoughts_token_count: 5, - tool_token_count: 2, - response_text: 'test-response', + describe('logCliConfiguration', () => { + it('should log the cli configuration', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + getModel: () => 'test-model', + getEmbeddingModel: () => 'test-embedding-model', + getSandbox: () => true, + getCoreTools: () => ['ls', 'read-file'], + getApprovalMode: () => 'default', + getContentGeneratorConfig: () => ({ + model: 'test-model', + apiKey: 'test-api-key', + vertexai: true, + codeAssist: false, + }), + getTelemetryLogUserPromptsEnabled: () => true, + getFileFilteringRespectGitIgnore: () => true, + getFileFilteringAllowBuildArtifacts: () => false, + getDebugMode: () => true, + getMcpServers: () => ({ + 'test-server': { + command: 'test-command', + }, + }), + getQuestion: () => 'test-question', + } as unknown as Config; + + logCliConfiguration(mockConfig); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'CLI configuration loaded.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.config', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + model: 'test-model', + embedding_model: 'test-embedding-model', + sandbox_enabled: true, + core_tools_enabled: 'ls,read-file', + approval_mode: 'default', + api_key_enabled: true, + vertex_ai_enabled: true, + code_assist_enabled: false, + log_user_prompts_enabled: true, + file_filtering_respect_git_ignore: true, + file_filtering_allow_build_artifacts: false, + debug_mode: true, + mcp_servers: 'test-server', + }, + }); + }); + }); + + describe('logApiResponse', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + } as Config; + + const mockMetrics = { + recordApiResponseMetrics: vi.fn(), + recordTokenUsageMetrics: vi.fn(), }; - logApiResponse(mockConfig, event); + beforeEach(() => { + vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation( + mockMetrics.recordApiResponseMetrics, + ); + vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation( + mockMetrics.recordTokenUsageMetrics, + ); + }); - expect(mockLogger.emit).toHaveBeenCalledWith({ - body: 'API response from test-model. Status: 200. Duration: 100ms.', - attributes: { - 'session.id': 'test-session-id', - 'event.name': EVENT_API_RESPONSE, - 'event.timestamp': '2025-01-01T00:00:00.000Z', - [SemanticAttributes.HTTP_STATUS_CODE]: 200, + it('should log an API response with all fields', () => { + const event = { model: 'test-model', status_code: 200, duration_ms: 100, @@ -71,49 +116,70 @@ describe('logApiResponse', () => { thoughts_token_count: 5, tool_token_count: 2, response_text: 'test-response', - }, + }; + + logApiResponse(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'API response from test-model. Status: 200. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': EVENT_API_RESPONSE, + 'event.timestamp': '2025-01-01T00:00:00.000Z', + [SemanticAttributes.HTTP_STATUS_CODE]: 200, + model: 'test-model', + status_code: 200, + duration_ms: 100, + attempt: 1, + output_token_count: 50, + cached_content_token_count: 10, + thoughts_token_count: 5, + tool_token_count: 2, + response_text: 'test-response', + }, + }); + + expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-model', + 100, + 200, + undefined, + ); + + expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-model', + 50, + 'output', + ); }); - expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith( - mockConfig, - 'test-model', - 100, - 200, - undefined, - ); + it('should log an API response with an error', () => { + const event = { + model: 'test-model', + duration_ms: 100, + attempt: 1, + error: 'test-error', + output_token_count: 50, + cached_content_token_count: 10, + thoughts_token_count: 5, + tool_token_count: 2, + response_text: 'test-response', + }; - expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith( - mockConfig, - 'test-model', - 50, - 'output', - ); - }); + logApiResponse(mockConfig, event); - it('should log an API response with an error', () => { - const event = { - model: 'test-model', - duration_ms: 100, - attempt: 1, - error: 'test-error', - output_token_count: 50, - cached_content_token_count: 10, - thoughts_token_count: 5, - tool_token_count: 2, - response_text: 'test-response', - }; - - logApiResponse(mockConfig, event); - - expect(mockLogger.emit).toHaveBeenCalledWith({ - body: 'API response from test-model. Status: N/A. Duration: 100ms.', - attributes: { - 'session.id': 'test-session-id', - ...event, - 'event.name': EVENT_API_RESPONSE, - 'event.timestamp': '2025-01-01T00:00:00.000Z', - 'error.message': 'test-error', - }, + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'API response from test-model. Status: N/A. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + ...event, + 'event.name': EVENT_API_RESPONSE, + 'event.timestamp': '2025-01-01T00:00:00.000Z', + 'error.message': 'test-error', + }, + }); }); }); }); diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index 48275829..c4f773b4 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -43,21 +43,28 @@ function getCommonAttributes(config: Config): LogAttributes { export function logCliConfiguration(config: Config): void { if (!isTelemetrySdkInitialized()) return; + const generatorConfig = config.getContentGeneratorConfig(); + const mcpServers = config.getMcpServers(); const attributes: LogAttributes = { ...getCommonAttributes(config), 'event.name': EVENT_CLI_CONFIG, 'event.timestamp': new Date().toISOString(), model: config.getModel(), + embedding_model: config.getEmbeddingModel(), sandbox_enabled: typeof config.getSandbox() === 'string' ? true : config.getSandbox(), core_tools_enabled: (config.getCoreTools() ?? []).join(','), approval_mode: config.getApprovalMode(), - vertex_ai_enabled: !!config.getContentGeneratorConfig().vertexai, + api_key_enabled: !!generatorConfig.apiKey, + vertex_ai_enabled: !!generatorConfig.vertexai, + code_assist_enabled: !!generatorConfig.codeAssist, log_user_prompts_enabled: config.getTelemetryLogUserPromptsEnabled(), file_filtering_respect_git_ignore: config.getFileFilteringRespectGitIgnore(), file_filtering_allow_build_artifacts: config.getFileFilteringAllowBuildArtifacts(), + debug_mode: config.getDebugMode(), + mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '', }; const logger = logs.getLogger(SERVICE_NAME); const logRecord: LogRecord = {