From 58c2925624c5befe130fbe120707455daf91c910 Mon Sep 17 00:00:00 2001 From: agarwalravikant Date: Tue, 19 Aug 2025 10:55:47 +0530 Subject: [PATCH] Changes to add tool_type as dimension (#6538) Co-authored-by: Ravikant Agarwal --- docs/telemetry.md | 1 + packages/cli/src/zed-integration/zedIntegration.ts | 9 +++++++++ packages/core/src/core/nonInteractiveToolExecutor.ts | 10 ++++++++++ .../src/telemetry/clearcut-logger/clearcut-logger.ts | 4 ++++ .../telemetry/clearcut-logger/event-metadata-key.ts | 3 +++ packages/core/src/telemetry/loggers.test.ts | 10 ++++++++++ packages/core/src/telemetry/loggers.ts | 1 + packages/core/src/telemetry/metrics.ts | 2 ++ packages/core/src/telemetry/types.ts | 6 ++++++ 9 files changed, 46 insertions(+) diff --git a/docs/telemetry.md b/docs/telemetry.md index ed094ce6..205242e7 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -240,6 +240,7 @@ Metrics are numerical measurements of behavior over time. The following metrics - `function_name` - `success` (boolean) - `decision` (string: "accept", "reject", or "modify", if applicable) + - `tool_type` (string: "mcp", or "native", if applicable) - `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency. - **Attributes**: diff --git a/packages/cli/src/zed-integration/zedIntegration.ts b/packages/cli/src/zed-integration/zedIntegration.ts index 6adaeb70..51b1f170 100644 --- a/packages/cli/src/zed-integration/zedIntegration.ts +++ b/packages/cli/src/zed-integration/zedIntegration.ts @@ -22,6 +22,7 @@ import { isWithinRoot, getErrorStatus, MCPServerConfig, + DiscoveredMCPTool, } from '@google/gemini-cli-core'; import * as acp from './acp.js'; import { AcpFileSystemService } from './fileSystemService.js'; @@ -344,6 +345,10 @@ class Session { duration_ms: durationMs, success: false, error: error.message, + tool_type: + typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool + ? 'mcp' + : 'native', }); return [ @@ -457,6 +462,10 @@ class Session { duration_ms: durationMs, success: true, prompt_id: promptId, + tool_type: + typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool + ? 'mcp' + : 'native', }); return convertToFunctionResponse(fc.name, callId, toolResult.llmContent); diff --git a/packages/core/src/core/nonInteractiveToolExecutor.ts b/packages/core/src/core/nonInteractiveToolExecutor.ts index b7c4faae..3849d52a 100644 --- a/packages/core/src/core/nonInteractiveToolExecutor.ts +++ b/packages/core/src/core/nonInteractiveToolExecutor.ts @@ -13,6 +13,7 @@ import { ToolRegistry, ToolResult, } from '../index.js'; +import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import { Config } from '../config/config.js'; import { convertToFunctionResponse } from './coreToolScheduler.js'; import { ToolCallDecision } from '../telemetry/tool-call-decision.js'; @@ -44,6 +45,7 @@ export async function executeToolCall( success: false, error: error.message, prompt_id: toolCallRequest.prompt_id, + tool_type: 'native', }); // Ensure the response structure matches what the API expects for an error return { @@ -109,6 +111,10 @@ export async function executeToolCall( prompt_id: toolCallRequest.prompt_id, metadata, decision: ToolCallDecision.AUTO_ACCEPT, + tool_type: + typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool + ? 'mcp' + : 'native', }); const response = convertToFunctionResponse( @@ -141,6 +147,10 @@ export async function executeToolCall( error: error.message, error_type: ToolErrorType.UNHANDLED_EXCEPTION, prompt_id: toolCallRequest.prompt_id, + tool_type: + typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool + ? 'mcp' + : 'native', }); return { callId: toolCallRequest.callId, diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 14551824..7777fa24 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -450,6 +450,10 @@ export class ClearcutLogger { gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE, value: JSON.stringify(event.error_type), }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_TYPE, + value: JSON.stringify(event.tool_type), + }, ]; if (event.metadata) { diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index d9fa16a9..34c8d9b0 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -234,4 +234,7 @@ export enum EventMetadataKey { // Logs the number of tokens after context window compression. GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61, + + // Logs tool type whether it is mcp or native. + GEMINI_CLI_TOOL_TYPE = 62, } diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index a8c5ed88..8097b4a4 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -524,6 +524,7 @@ describe('loggers', () => { success: true, decision: ToolCallDecision.ACCEPT, prompt_id: 'prompt-id-1', + tool_type: 'native', }, }); @@ -533,6 +534,7 @@ describe('loggers', () => { 100, true, ToolCallDecision.ACCEPT, + 'native', ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -587,6 +589,7 @@ describe('loggers', () => { success: false, decision: ToolCallDecision.REJECT, prompt_id: 'prompt-id-2', + tool_type: 'native', }, }); @@ -596,6 +599,7 @@ describe('loggers', () => { 100, false, ToolCallDecision.REJECT, + 'native', ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -653,6 +657,7 @@ describe('loggers', () => { success: true, decision: ToolCallDecision.MODIFY, prompt_id: 'prompt-id-3', + tool_type: 'native', }, }); @@ -662,6 +667,7 @@ describe('loggers', () => { 100, true, ToolCallDecision.MODIFY, + 'native', ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -717,6 +723,7 @@ describe('loggers', () => { duration_ms: 100, success: true, prompt_id: 'prompt-id-4', + tool_type: 'native', }, }); @@ -726,6 +733,7 @@ describe('loggers', () => { 100, true, undefined, + 'native', ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -786,6 +794,7 @@ describe('loggers', () => { error_type: ToolErrorType.UNKNOWN, 'error.type': ToolErrorType.UNKNOWN, prompt_id: 'prompt-id-5', + tool_type: 'native', }, }); @@ -795,6 +804,7 @@ describe('loggers', () => { 100, false, undefined, + 'native', ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index bfa98b32..6504c0e7 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -148,6 +148,7 @@ export function logToolCall(config: Config, event: ToolCallEvent): void { event.duration_ms, event.success, event.decision, + event.tool_type, ); } diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index c1125086..4abcb4f0 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -119,6 +119,7 @@ export function recordToolCallMetrics( durationMs: number, success: boolean, decision?: 'accept' | 'reject' | 'modify' | 'auto_accept', + tool_type?: 'native' | 'mcp', ): void { if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized) return; @@ -128,6 +129,7 @@ export function recordToolCallMetrics( function_name: functionName, success, decision, + tool_type, }; toolCallCounter.add(1, metricAttributes); toolCallLatencyHistogram.record(durationMs, { diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index d5dda177..0a49ca02 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -7,6 +7,7 @@ import { GenerateContentResponseUsageMetadata } from '@google/genai'; import { Config } from '../config/config.js'; import { CompletedToolCall } from '../core/coreToolScheduler.js'; +import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import { FileDiff } from '../tools/tools.js'; import { AuthType } from '../core/contentGenerator.js'; import { @@ -114,6 +115,7 @@ export class ToolCallEvent implements BaseTelemetryEvent { error?: string; error_type?: string; prompt_id: string; + tool_type: 'native' | 'mcp'; // eslint-disable-next-line @typescript-eslint/no-explicit-any metadata?: { [key: string]: any }; @@ -130,6 +132,10 @@ export class ToolCallEvent implements BaseTelemetryEvent { this.error = call.response.error?.message; this.error_type = call.response.errorType; this.prompt_id = call.request.prompt_id; + this.tool_type = + typeof call.tool !== 'undefined' && call.tool instanceof DiscoveredMCPTool + ? 'mcp' + : 'native'; if ( call.status === 'success' &&