Changes to add tool_type as dimension (#6538)

Co-authored-by: Ravikant Agarwal <ravikantag@google.com>
This commit is contained in:
agarwalravikant 2025-08-19 10:55:47 +05:30 committed by GitHub
parent e290a61a52
commit 58c2925624
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 46 additions and 0 deletions

View File

@ -240,6 +240,7 @@ Metrics are numerical measurements of behavior over time. The following metrics
- `function_name` - `function_name`
- `success` (boolean) - `success` (boolean)
- `decision` (string: "accept", "reject", or "modify", if applicable) - `decision` (string: "accept", "reject", or "modify", if applicable)
- `tool_type` (string: "mcp", or "native", if applicable)
- `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency. - `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency.
- **Attributes**: - **Attributes**:

View File

@ -22,6 +22,7 @@ import {
isWithinRoot, isWithinRoot,
getErrorStatus, getErrorStatus,
MCPServerConfig, MCPServerConfig,
DiscoveredMCPTool,
} from '@google/gemini-cli-core'; } from '@google/gemini-cli-core';
import * as acp from './acp.js'; import * as acp from './acp.js';
import { AcpFileSystemService } from './fileSystemService.js'; import { AcpFileSystemService } from './fileSystemService.js';
@ -344,6 +345,10 @@ class Session {
duration_ms: durationMs, duration_ms: durationMs,
success: false, success: false,
error: error.message, error: error.message,
tool_type:
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native',
}); });
return [ return [
@ -457,6 +462,10 @@ class Session {
duration_ms: durationMs, duration_ms: durationMs,
success: true, success: true,
prompt_id: promptId, prompt_id: promptId,
tool_type:
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native',
}); });
return convertToFunctionResponse(fc.name, callId, toolResult.llmContent); return convertToFunctionResponse(fc.name, callId, toolResult.llmContent);

View File

@ -13,6 +13,7 @@ import {
ToolRegistry, ToolRegistry,
ToolResult, ToolResult,
} from '../index.js'; } from '../index.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { Config } from '../config/config.js'; import { Config } from '../config/config.js';
import { convertToFunctionResponse } from './coreToolScheduler.js'; import { convertToFunctionResponse } from './coreToolScheduler.js';
import { ToolCallDecision } from '../telemetry/tool-call-decision.js'; import { ToolCallDecision } from '../telemetry/tool-call-decision.js';
@ -44,6 +45,7 @@ export async function executeToolCall(
success: false, success: false,
error: error.message, error: error.message,
prompt_id: toolCallRequest.prompt_id, prompt_id: toolCallRequest.prompt_id,
tool_type: 'native',
}); });
// Ensure the response structure matches what the API expects for an error // Ensure the response structure matches what the API expects for an error
return { return {
@ -109,6 +111,10 @@ export async function executeToolCall(
prompt_id: toolCallRequest.prompt_id, prompt_id: toolCallRequest.prompt_id,
metadata, metadata,
decision: ToolCallDecision.AUTO_ACCEPT, decision: ToolCallDecision.AUTO_ACCEPT,
tool_type:
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native',
}); });
const response = convertToFunctionResponse( const response = convertToFunctionResponse(
@ -141,6 +147,10 @@ export async function executeToolCall(
error: error.message, error: error.message,
error_type: ToolErrorType.UNHANDLED_EXCEPTION, error_type: ToolErrorType.UNHANDLED_EXCEPTION,
prompt_id: toolCallRequest.prompt_id, prompt_id: toolCallRequest.prompt_id,
tool_type:
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native',
}); });
return { return {
callId: toolCallRequest.callId, callId: toolCallRequest.callId,

View File

@ -450,6 +450,10 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE, gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
value: JSON.stringify(event.error_type), value: JSON.stringify(event.error_type),
}, },
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_TYPE,
value: JSON.stringify(event.tool_type),
},
]; ];
if (event.metadata) { if (event.metadata) {

View File

@ -234,4 +234,7 @@ export enum EventMetadataKey {
// Logs the number of tokens after context window compression. // Logs the number of tokens after context window compression.
GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61, GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61,
// Logs tool type whether it is mcp or native.
GEMINI_CLI_TOOL_TYPE = 62,
} }

View File

@ -524,6 +524,7 @@ describe('loggers', () => {
success: true, success: true,
decision: ToolCallDecision.ACCEPT, decision: ToolCallDecision.ACCEPT,
prompt_id: 'prompt-id-1', prompt_id: 'prompt-id-1',
tool_type: 'native',
}, },
}); });
@ -533,6 +534,7 @@ describe('loggers', () => {
100, 100,
true, true,
ToolCallDecision.ACCEPT, ToolCallDecision.ACCEPT,
'native',
); );
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@ -587,6 +589,7 @@ describe('loggers', () => {
success: false, success: false,
decision: ToolCallDecision.REJECT, decision: ToolCallDecision.REJECT,
prompt_id: 'prompt-id-2', prompt_id: 'prompt-id-2',
tool_type: 'native',
}, },
}); });
@ -596,6 +599,7 @@ describe('loggers', () => {
100, 100,
false, false,
ToolCallDecision.REJECT, ToolCallDecision.REJECT,
'native',
); );
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@ -653,6 +657,7 @@ describe('loggers', () => {
success: true, success: true,
decision: ToolCallDecision.MODIFY, decision: ToolCallDecision.MODIFY,
prompt_id: 'prompt-id-3', prompt_id: 'prompt-id-3',
tool_type: 'native',
}, },
}); });
@ -662,6 +667,7 @@ describe('loggers', () => {
100, 100,
true, true,
ToolCallDecision.MODIFY, ToolCallDecision.MODIFY,
'native',
); );
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@ -717,6 +723,7 @@ describe('loggers', () => {
duration_ms: 100, duration_ms: 100,
success: true, success: true,
prompt_id: 'prompt-id-4', prompt_id: 'prompt-id-4',
tool_type: 'native',
}, },
}); });
@ -726,6 +733,7 @@ describe('loggers', () => {
100, 100,
true, true,
undefined, undefined,
'native',
); );
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@ -786,6 +794,7 @@ describe('loggers', () => {
error_type: ToolErrorType.UNKNOWN, error_type: ToolErrorType.UNKNOWN,
'error.type': ToolErrorType.UNKNOWN, 'error.type': ToolErrorType.UNKNOWN,
prompt_id: 'prompt-id-5', prompt_id: 'prompt-id-5',
tool_type: 'native',
}, },
}); });
@ -795,6 +804,7 @@ describe('loggers', () => {
100, 100,
false, false,
undefined, undefined,
'native',
); );
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ expect(mockUiEvent.addEvent).toHaveBeenCalledWith({

View File

@ -148,6 +148,7 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
event.duration_ms, event.duration_ms,
event.success, event.success,
event.decision, event.decision,
event.tool_type,
); );
} }

View File

@ -119,6 +119,7 @@ export function recordToolCallMetrics(
durationMs: number, durationMs: number,
success: boolean, success: boolean,
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept', decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
tool_type?: 'native' | 'mcp',
): void { ): void {
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized) if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
return; return;
@ -128,6 +129,7 @@ export function recordToolCallMetrics(
function_name: functionName, function_name: functionName,
success, success,
decision, decision,
tool_type,
}; };
toolCallCounter.add(1, metricAttributes); toolCallCounter.add(1, metricAttributes);
toolCallLatencyHistogram.record(durationMs, { toolCallLatencyHistogram.record(durationMs, {

View File

@ -7,6 +7,7 @@
import { GenerateContentResponseUsageMetadata } from '@google/genai'; import { GenerateContentResponseUsageMetadata } from '@google/genai';
import { Config } from '../config/config.js'; import { Config } from '../config/config.js';
import { CompletedToolCall } from '../core/coreToolScheduler.js'; import { CompletedToolCall } from '../core/coreToolScheduler.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { FileDiff } from '../tools/tools.js'; import { FileDiff } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js'; import { AuthType } from '../core/contentGenerator.js';
import { import {
@ -114,6 +115,7 @@ export class ToolCallEvent implements BaseTelemetryEvent {
error?: string; error?: string;
error_type?: string; error_type?: string;
prompt_id: string; prompt_id: string;
tool_type: 'native' | 'mcp';
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
metadata?: { [key: string]: any }; metadata?: { [key: string]: any };
@ -130,6 +132,10 @@ export class ToolCallEvent implements BaseTelemetryEvent {
this.error = call.response.error?.message; this.error = call.response.error?.message;
this.error_type = call.response.errorType; this.error_type = call.response.errorType;
this.prompt_id = call.request.prompt_id; this.prompt_id = call.request.prompt_id;
this.tool_type =
typeof call.tool !== 'undefined' && call.tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native';
if ( if (
call.status === 'success' && call.status === 'success' &&