From 714421c2da4f5d6b9c1c7060fdf5c47ba1c965ca Mon Sep 17 00:00:00 2001 From: Jerop Kipruto Date: Sun, 15 Jun 2025 16:24:53 -0400 Subject: [PATCH] Add file operation telemetry (#1068) Introduces telemetry for file create, read, and update operations. This change adds the `gemini_cli.file.operation.count` metric, recorded by the `read-file`, `read-many-files`, and `write-file` tools. The metric includes the following attributes: - `operation` (string: `create`, `read`, `update`): The type of file operation. - `lines` (optional, Int): Number of lines in the file. - `mimetype` (optional, string): Mimetype of the file. - `extension` (optional, string): File extension of the file. Here is a stacked bar chart of file operations by extension (`js`, `ts`, `md`): ![image](https://github.com/user-attachments/assets/3e8f8ea9-6155-4186-863c-075cc47647c5) Here is a stacked bar chart of file operations by type (`create`, `read`, `update`): ![image](https://github.com/user-attachments/assets/3fcf491d-31d0-4ba8-80e6-7fd2bd9c7c27) #750 cc @allenhutchison as discussed --- docs/core/telemetry.md | 9 + packages/core/src/telemetry/constants.ts | 1 + packages/core/src/telemetry/metrics.test.ts | 198 ++++++++++++++++---- packages/core/src/telemetry/metrics.ts | 31 ++- packages/core/src/tools/read-file.ts | 18 ++ packages/core/src/tools/read-many-files.ts | 17 ++ packages/core/src/tools/write-file.ts | 26 +++ packages/core/src/utils/fileUtils.ts | 10 + 8 files changed, 274 insertions(+), 36 deletions(-) diff --git a/docs/core/telemetry.md b/docs/core/telemetry.md index c42c2ed6..42719db5 100644 --- a/docs/core/telemetry.md +++ b/docs/core/telemetry.md @@ -273,6 +273,15 @@ These are numerical measurements of behavior over time. - `model` - `gemini_cli.token.usage` (Counter, Int): Counts the number of tokens used. + - **Attributes**: - `model` - `type` (string: "input", "output", "thought", "cache", or "tool") + +- `gemini_cli.file.operation.count` (Counter, Int): Counts file operations. + + - **Attributes**: + - `operation` (string: "create", "read", "update"): The type of file operation. + - `lines` (optional, Int): Number of lines in the file. + - `mimetype` (optional, string): Mimetype of the file. + - `extension` (optional, string): File extension of the file. diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts index bbcdbada..de760205 100644 --- a/packages/core/src/telemetry/constants.ts +++ b/packages/core/src/telemetry/constants.ts @@ -19,3 +19,4 @@ export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count'; export const METRIC_API_REQUEST_LATENCY = 'gemini_cli.api.request.latency'; export const METRIC_TOKEN_USAGE = 'gemini_cli.token.usage'; export const METRIC_SESSION_COUNT = 'gemini_cli.session.count'; +export const METRIC_FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count'; diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts index 7e24b9ad..4fcdd9e1 100644 --- a/packages/core/src/telemetry/metrics.test.ts +++ b/packages/core/src/telemetry/metrics.test.ts @@ -5,32 +5,84 @@ */ import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest'; -import { Counter, Meter, metrics } from '@opentelemetry/api'; -import { initializeMetrics, recordTokenUsageMetrics } from './metrics.js'; +import type { + Counter, + Meter, + Attributes, + Context, + Histogram, +} from '@opentelemetry/api'; import { Config } from '../config/config.js'; +import { FileOperation } from './metrics.js'; -const mockCounter = { - add: vi.fn(), +const mockCounterAddFn: Mock< + (value: number, attributes?: Attributes, context?: Context) => void +> = vi.fn(); +const mockHistogramRecordFn: Mock< + (value: number, attributes?: Attributes, context?: Context) => void +> = vi.fn(); + +const mockCreateCounterFn: Mock<(name: string, options?: unknown) => Counter> = + vi.fn(); +const mockCreateHistogramFn: Mock< + (name: string, options?: unknown) => Histogram +> = vi.fn(); + +const mockCounterInstance = { + add: mockCounterAddFn, } as unknown as Counter; -const mockMeter = { - createCounter: vi.fn().mockReturnValue(mockCounter), - createHistogram: vi.fn().mockReturnValue({ record: vi.fn() }), +const mockHistogramInstance = { + record: mockHistogramRecordFn, +} as unknown as Histogram; + +const mockMeterInstance = { + createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance), + createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance), } as unknown as Meter; -vi.mock('@opentelemetry/api', () => ({ - metrics: { - getMeter: vi.fn(), - }, - ValueType: { - INT: 1, - }, -})); +function originalOtelMockFactory() { + return { + metrics: { + getMeter: vi.fn(), + }, + ValueType: { + INT: 1, + }, + }; +} + +vi.mock('@opentelemetry/api', originalOtelMockFactory); describe('Telemetry Metrics', () => { - beforeEach(() => { - vi.clearAllMocks(); - (metrics.getMeter as Mock).mockReturnValue(mockMeter); + let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics; + let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics; + let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric; + + beforeEach(async () => { + vi.resetModules(); + vi.doMock('@opentelemetry/api', () => { + const actualApi = originalOtelMockFactory(); + (actualApi.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance); + return actualApi; + }); + + const metricsJsModule = await import('./metrics.js'); + initializeMetricsModule = metricsJsModule.initializeMetrics; + recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics; + recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric; + + const otelApiModule = await import('@opentelemetry/api'); + + mockCounterAddFn.mockClear(); + mockCreateCounterFn.mockClear(); + mockCreateHistogramFn.mockClear(); + mockHistogramRecordFn.mockClear(); + (otelApiModule.metrics.getMeter as Mock).mockClear(); + + (otelApiModule.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance); + mockCreateCounterFn.mockReturnValue(mockCounterInstance); + mockCreateHistogramFn.mockReturnValue(mockHistogramInstance); }); describe('recordTokenUsageMetrics', () => { @@ -39,14 +91,18 @@ describe('Telemetry Metrics', () => { } as unknown as Config; it('should not record metrics if not initialized', () => { - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 100, 'input'); - expect(mockCounter.add).not.toHaveBeenCalled(); + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input'); + expect(mockCounterAddFn).not.toHaveBeenCalled(); }); it('should record token usage with the correct attributes', () => { - initializeMetrics(mockConfig); - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 100, 'input'); - expect(mockCounter.add).toHaveBeenCalledWith(100, { + initializeMetricsModule(mockConfig); + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input'); + expect(mockCounterAddFn).toHaveBeenCalledTimes(2); + expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { + 'session.id': 'test-session-id', + }); + expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 100, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'input', @@ -54,30 +110,32 @@ describe('Telemetry Metrics', () => { }); it('should record token usage for different types', () => { - initializeMetrics(mockConfig); - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 50, 'output'); - expect(mockCounter.add).toHaveBeenCalledWith(50, { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 50, 'output'); + expect(mockCounterAddFn).toHaveBeenCalledWith(50, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'output', }); - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 25, 'thought'); - expect(mockCounter.add).toHaveBeenCalledWith(25, { + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 25, 'thought'); + expect(mockCounterAddFn).toHaveBeenCalledWith(25, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'thought', }); - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 75, 'cache'); - expect(mockCounter.add).toHaveBeenCalledWith(75, { + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 75, 'cache'); + expect(mockCounterAddFn).toHaveBeenCalledWith(75, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'cache', }); - recordTokenUsageMetrics(mockConfig, 'gemini-pro', 125, 'tool'); - expect(mockCounter.add).toHaveBeenCalledWith(125, { + recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 125, 'tool'); + expect(mockCounterAddFn).toHaveBeenCalledWith(125, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'tool', @@ -85,13 +143,83 @@ describe('Telemetry Metrics', () => { }); it('should handle different models', () => { - initializeMetrics(mockConfig); - recordTokenUsageMetrics(mockConfig, 'gemini-ultra', 200, 'input'); - expect(mockCounter.add).toHaveBeenCalledWith(200, { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + + recordTokenUsageMetricsModule(mockConfig, 'gemini-ultra', 200, 'input'); + expect(mockCounterAddFn).toHaveBeenCalledWith(200, { 'session.id': 'test-session-id', model: 'gemini-ultra', type: 'input', }); }); }); + + describe('recordFileOperationMetric', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + } as unknown as Config; + + it('should not record metrics if not initialized', () => { + recordFileOperationMetricModule( + mockConfig, + FileOperation.CREATE, + 10, + 'text/plain', + 'txt', + ); + expect(mockCounterAddFn).not.toHaveBeenCalled(); + }); + + it('should record file creation with all attributes', () => { + initializeMetricsModule(mockConfig); + recordFileOperationMetricModule( + mockConfig, + FileOperation.CREATE, + 10, + 'text/plain', + 'txt', + ); + + expect(mockCounterAddFn).toHaveBeenCalledTimes(2); + expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { + 'session.id': 'test-session-id', + }); + expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, { + 'session.id': 'test-session-id', + operation: FileOperation.CREATE, + lines: 10, + mimetype: 'text/plain', + extension: 'txt', + }); + }); + + it('should record file read with minimal attributes', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + + recordFileOperationMetricModule(mockConfig, FileOperation.READ); + expect(mockCounterAddFn).toHaveBeenCalledWith(1, { + 'session.id': 'test-session-id', + operation: FileOperation.READ, + }); + }); + + it('should record file update with some attributes', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + + recordFileOperationMetricModule( + mockConfig, + FileOperation.UPDATE, + undefined, + 'application/javascript', + ); + expect(mockCounterAddFn).toHaveBeenCalledWith(1, { + 'session.id': 'test-session-id', + operation: FileOperation.UPDATE, + mimetype: 'application/javascript', + }); + }); + }); }); diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index 59979ef3..124bc602 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -20,15 +20,23 @@ import { METRIC_API_REQUEST_LATENCY, METRIC_TOKEN_USAGE, METRIC_SESSION_COUNT, + METRIC_FILE_OPERATION_COUNT, } from './constants.js'; import { Config } from '../config/config.js'; +export enum FileOperation { + CREATE = 'create', + READ = 'read', + UPDATE = 'update', +} + let cliMeter: Meter | undefined; let toolCallCounter: Counter | undefined; let toolCallLatencyHistogram: Histogram | undefined; let apiRequestCounter: Counter | undefined; let apiRequestLatencyHistogram: Histogram | undefined; let tokenUsageCounter: Counter | undefined; +let fileOperationCounter: Counter | undefined; let isMetricsInitialized = false; function getCommonAttributes(config: Config): Attributes { @@ -75,7 +83,10 @@ export function initializeMetrics(config: Config): void { description: 'Counts the total number of tokens used.', valueType: ValueType.INT, }); - + fileOperationCounter = meter.createCounter(METRIC_FILE_OPERATION_COUNT, { + description: 'Counts file operations (create, read, update).', + valueType: ValueType.INT, + }); const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, { description: 'Count of CLI sessions started.', valueType: ValueType.INT, @@ -171,3 +182,21 @@ export function recordApiErrorMetrics( model, }); } + +export function recordFileOperationMetric( + config: Config, + operation: FileOperation, + lines?: number, + mimetype?: string, + extension?: string, +): void { + if (!fileOperationCounter || !isMetricsInitialized) return; + const attributes: Attributes = { + ...getCommonAttributes(config), + operation, + }; + if (lines !== undefined) attributes.lines = lines; + if (mimetype !== undefined) attributes.mimetype = mimetype; + if (extension !== undefined) attributes.extension = extension; + fileOperationCounter.add(1, attributes); +} diff --git a/packages/core/src/tools/read-file.ts b/packages/core/src/tools/read-file.ts index 586a7123..5cf49209 100644 --- a/packages/core/src/tools/read-file.ts +++ b/packages/core/src/tools/read-file.ts @@ -10,6 +10,11 @@ import { makeRelative, shortenPath } from '../utils/paths.js'; import { BaseTool, ToolResult } from './tools.js'; import { isWithinRoot, processSingleFileContent } from '../utils/fileUtils.js'; import { Config } from '../config/config.js'; +import { getSpecificMimeType } from '../utils/fileUtils.js'; +import { + recordFileOperationMetric, + FileOperation, +} from '../telemetry/metrics.js'; /** * Parameters for the ReadFile tool @@ -145,6 +150,19 @@ export class ReadFileTool extends BaseTool { }; } + const lines = + typeof result.llmContent === 'string' + ? result.llmContent.split('\n').length + : undefined; + const mimetype = getSpecificMimeType(params.absolute_path); + recordFileOperationMetric( + this.config, + FileOperation.READ, + lines, + mimetype, + path.extname(params.absolute_path), + ); + return { llmContent: result.llmContent, returnDisplay: result.returnDisplay, diff --git a/packages/core/src/tools/read-many-files.ts b/packages/core/src/tools/read-many-files.ts index 107e16b3..62430c10 100644 --- a/packages/core/src/tools/read-many-files.ts +++ b/packages/core/src/tools/read-many-files.ts @@ -14,9 +14,14 @@ import { detectFileType, processSingleFileContent, DEFAULT_ENCODING, + getSpecificMimeType, } from '../utils/fileUtils.js'; import { PartListUnion } from '@google/genai'; import { Config } from '../config/config.js'; +import { + recordFileOperationMetric, + FileOperation, +} from '../telemetry/metrics.js'; /** * Parameters for the ReadManyFilesTool. @@ -420,6 +425,18 @@ Use this tool when the user's query implies needing the content of several files contentParts.push(fileReadResult.llmContent); // This is a Part for image/pdf } processedFilesRelativePaths.push(relativePathForDisplay); + const lines = + typeof fileReadResult.llmContent === 'string' + ? fileReadResult.llmContent.split('\n').length + : undefined; + const mimetype = getSpecificMimeType(filePath); + recordFileOperationMetric( + this.config, + FileOperation.READ, + lines, + mimetype, + path.extname(filePath), + ); } } diff --git a/packages/core/src/tools/write-file.ts b/packages/core/src/tools/write-file.ts index b9e07034..b19b00ac 100644 --- a/packages/core/src/tools/write-file.ts +++ b/packages/core/src/tools/write-file.ts @@ -26,6 +26,11 @@ import { import { GeminiClient } from '../core/client.js'; import { DEFAULT_DIFF_OPTIONS } from './diffOptions.js'; import { ModifiableTool, ModifyContext } from './modifiable-tool.js'; +import { getSpecificMimeType } from '../utils/fileUtils.js'; +import { + recordFileOperationMetric, + FileOperation, +} from '../telemetry/metrics.js'; /** * Parameters for the WriteFile tool @@ -271,6 +276,27 @@ export class WriteFileTool const displayResult: FileDiff = { fileDiff, fileName }; + const lines = fileContent.split('\n').length; + const mimetype = getSpecificMimeType(params.file_path); + const extension = path.extname(params.file_path); // Get extension + if (isNewFile) { + recordFileOperationMetric( + this.config, + FileOperation.CREATE, + lines, + mimetype, + extension, + ); + } else { + recordFileOperationMetric( + this.config, + FileOperation.UPDATE, + lines, + mimetype, + extension, + ); + } + return { llmContent: llmSuccessMessage, returnDisplay: displayResult, diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index d726c053..cb4d333a 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -16,6 +16,16 @@ const MAX_LINE_LENGTH_TEXT_FILE = 2000; // Default values for encoding and separator format export const DEFAULT_ENCODING: BufferEncoding = 'utf-8'; +/** + * Looks up the specific MIME type for a file path. + * @param filePath Path to the file. + * @returns The specific MIME type string (e.g., 'text/python', 'application/javascript') or undefined if not found or ambiguous. + */ +export function getSpecificMimeType(filePath: string): string | undefined { + const lookedUpMime = mime.lookup(filePath); + return typeof lookedUpMime === 'string' ? lookedUpMime : undefined; +} + /** * Checks if a path is within a given root directory. * @param pathToCheck The absolute path to check.