Changes to add tool_type as dimension (#6538)
Co-authored-by: Ravikant Agarwal <ravikantag@google.com>
This commit is contained in:
parent
e290a61a52
commit
58c2925624
|
@ -240,6 +240,7 @@ Metrics are numerical measurements of behavior over time. The following metrics
|
||||||
- `function_name`
|
- `function_name`
|
||||||
- `success` (boolean)
|
- `success` (boolean)
|
||||||
- `decision` (string: "accept", "reject", or "modify", if applicable)
|
- `decision` (string: "accept", "reject", or "modify", if applicable)
|
||||||
|
- `tool_type` (string: "mcp", or "native", if applicable)
|
||||||
|
|
||||||
- `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency.
|
- `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency.
|
||||||
- **Attributes**:
|
- **Attributes**:
|
||||||
|
|
|
@ -22,6 +22,7 @@ import {
|
||||||
isWithinRoot,
|
isWithinRoot,
|
||||||
getErrorStatus,
|
getErrorStatus,
|
||||||
MCPServerConfig,
|
MCPServerConfig,
|
||||||
|
DiscoveredMCPTool,
|
||||||
} from '@google/gemini-cli-core';
|
} from '@google/gemini-cli-core';
|
||||||
import * as acp from './acp.js';
|
import * as acp from './acp.js';
|
||||||
import { AcpFileSystemService } from './fileSystemService.js';
|
import { AcpFileSystemService } from './fileSystemService.js';
|
||||||
|
@ -344,6 +345,10 @@ class Session {
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
success: false,
|
success: false,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
|
tool_type:
|
||||||
|
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
|
||||||
|
? 'mcp'
|
||||||
|
: 'native',
|
||||||
});
|
});
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
@ -457,6 +462,10 @@ class Session {
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
success: true,
|
success: true,
|
||||||
prompt_id: promptId,
|
prompt_id: promptId,
|
||||||
|
tool_type:
|
||||||
|
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
|
||||||
|
? 'mcp'
|
||||||
|
: 'native',
|
||||||
});
|
});
|
||||||
|
|
||||||
return convertToFunctionResponse(fc.name, callId, toolResult.llmContent);
|
return convertToFunctionResponse(fc.name, callId, toolResult.llmContent);
|
||||||
|
|
|
@ -13,6 +13,7 @@ import {
|
||||||
ToolRegistry,
|
ToolRegistry,
|
||||||
ToolResult,
|
ToolResult,
|
||||||
} from '../index.js';
|
} from '../index.js';
|
||||||
|
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { convertToFunctionResponse } from './coreToolScheduler.js';
|
import { convertToFunctionResponse } from './coreToolScheduler.js';
|
||||||
import { ToolCallDecision } from '../telemetry/tool-call-decision.js';
|
import { ToolCallDecision } from '../telemetry/tool-call-decision.js';
|
||||||
|
@ -44,6 +45,7 @@ export async function executeToolCall(
|
||||||
success: false,
|
success: false,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
prompt_id: toolCallRequest.prompt_id,
|
prompt_id: toolCallRequest.prompt_id,
|
||||||
|
tool_type: 'native',
|
||||||
});
|
});
|
||||||
// Ensure the response structure matches what the API expects for an error
|
// Ensure the response structure matches what the API expects for an error
|
||||||
return {
|
return {
|
||||||
|
@ -109,6 +111,10 @@ export async function executeToolCall(
|
||||||
prompt_id: toolCallRequest.prompt_id,
|
prompt_id: toolCallRequest.prompt_id,
|
||||||
metadata,
|
metadata,
|
||||||
decision: ToolCallDecision.AUTO_ACCEPT,
|
decision: ToolCallDecision.AUTO_ACCEPT,
|
||||||
|
tool_type:
|
||||||
|
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
|
||||||
|
? 'mcp'
|
||||||
|
: 'native',
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = convertToFunctionResponse(
|
const response = convertToFunctionResponse(
|
||||||
|
@ -141,6 +147,10 @@ export async function executeToolCall(
|
||||||
error: error.message,
|
error: error.message,
|
||||||
error_type: ToolErrorType.UNHANDLED_EXCEPTION,
|
error_type: ToolErrorType.UNHANDLED_EXCEPTION,
|
||||||
prompt_id: toolCallRequest.prompt_id,
|
prompt_id: toolCallRequest.prompt_id,
|
||||||
|
tool_type:
|
||||||
|
typeof tool !== 'undefined' && tool instanceof DiscoveredMCPTool
|
||||||
|
? 'mcp'
|
||||||
|
: 'native',
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
callId: toolCallRequest.callId,
|
callId: toolCallRequest.callId,
|
||||||
|
|
|
@ -450,6 +450,10 @@ export class ClearcutLogger {
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
|
||||||
value: JSON.stringify(event.error_type),
|
value: JSON.stringify(event.error_type),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_TYPE,
|
||||||
|
value: JSON.stringify(event.tool_type),
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
if (event.metadata) {
|
if (event.metadata) {
|
||||||
|
|
|
@ -234,4 +234,7 @@ export enum EventMetadataKey {
|
||||||
|
|
||||||
// Logs the number of tokens after context window compression.
|
// Logs the number of tokens after context window compression.
|
||||||
GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61,
|
GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61,
|
||||||
|
|
||||||
|
// Logs tool type whether it is mcp or native.
|
||||||
|
GEMINI_CLI_TOOL_TYPE = 62,
|
||||||
}
|
}
|
||||||
|
|
|
@ -524,6 +524,7 @@ describe('loggers', () => {
|
||||||
success: true,
|
success: true,
|
||||||
decision: ToolCallDecision.ACCEPT,
|
decision: ToolCallDecision.ACCEPT,
|
||||||
prompt_id: 'prompt-id-1',
|
prompt_id: 'prompt-id-1',
|
||||||
|
tool_type: 'native',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -533,6 +534,7 @@ describe('loggers', () => {
|
||||||
100,
|
100,
|
||||||
true,
|
true,
|
||||||
ToolCallDecision.ACCEPT,
|
ToolCallDecision.ACCEPT,
|
||||||
|
'native',
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||||
|
@ -587,6 +589,7 @@ describe('loggers', () => {
|
||||||
success: false,
|
success: false,
|
||||||
decision: ToolCallDecision.REJECT,
|
decision: ToolCallDecision.REJECT,
|
||||||
prompt_id: 'prompt-id-2',
|
prompt_id: 'prompt-id-2',
|
||||||
|
tool_type: 'native',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -596,6 +599,7 @@ describe('loggers', () => {
|
||||||
100,
|
100,
|
||||||
false,
|
false,
|
||||||
ToolCallDecision.REJECT,
|
ToolCallDecision.REJECT,
|
||||||
|
'native',
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||||
|
@ -653,6 +657,7 @@ describe('loggers', () => {
|
||||||
success: true,
|
success: true,
|
||||||
decision: ToolCallDecision.MODIFY,
|
decision: ToolCallDecision.MODIFY,
|
||||||
prompt_id: 'prompt-id-3',
|
prompt_id: 'prompt-id-3',
|
||||||
|
tool_type: 'native',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -662,6 +667,7 @@ describe('loggers', () => {
|
||||||
100,
|
100,
|
||||||
true,
|
true,
|
||||||
ToolCallDecision.MODIFY,
|
ToolCallDecision.MODIFY,
|
||||||
|
'native',
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||||
|
@ -717,6 +723,7 @@ describe('loggers', () => {
|
||||||
duration_ms: 100,
|
duration_ms: 100,
|
||||||
success: true,
|
success: true,
|
||||||
prompt_id: 'prompt-id-4',
|
prompt_id: 'prompt-id-4',
|
||||||
|
tool_type: 'native',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -726,6 +733,7 @@ describe('loggers', () => {
|
||||||
100,
|
100,
|
||||||
true,
|
true,
|
||||||
undefined,
|
undefined,
|
||||||
|
'native',
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||||
|
@ -786,6 +794,7 @@ describe('loggers', () => {
|
||||||
error_type: ToolErrorType.UNKNOWN,
|
error_type: ToolErrorType.UNKNOWN,
|
||||||
'error.type': ToolErrorType.UNKNOWN,
|
'error.type': ToolErrorType.UNKNOWN,
|
||||||
prompt_id: 'prompt-id-5',
|
prompt_id: 'prompt-id-5',
|
||||||
|
tool_type: 'native',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -795,6 +804,7 @@ describe('loggers', () => {
|
||||||
100,
|
100,
|
||||||
false,
|
false,
|
||||||
undefined,
|
undefined,
|
||||||
|
'native',
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||||
|
|
|
@ -148,6 +148,7 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
|
||||||
event.duration_ms,
|
event.duration_ms,
|
||||||
event.success,
|
event.success,
|
||||||
event.decision,
|
event.decision,
|
||||||
|
event.tool_type,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -119,6 +119,7 @@ export function recordToolCallMetrics(
|
||||||
durationMs: number,
|
durationMs: number,
|
||||||
success: boolean,
|
success: boolean,
|
||||||
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
|
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
|
||||||
|
tool_type?: 'native' | 'mcp',
|
||||||
): void {
|
): void {
|
||||||
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
|
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
|
||||||
return;
|
return;
|
||||||
|
@ -128,6 +129,7 @@ export function recordToolCallMetrics(
|
||||||
function_name: functionName,
|
function_name: functionName,
|
||||||
success,
|
success,
|
||||||
decision,
|
decision,
|
||||||
|
tool_type,
|
||||||
};
|
};
|
||||||
toolCallCounter.add(1, metricAttributes);
|
toolCallCounter.add(1, metricAttributes);
|
||||||
toolCallLatencyHistogram.record(durationMs, {
|
toolCallLatencyHistogram.record(durationMs, {
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { CompletedToolCall } from '../core/coreToolScheduler.js';
|
import { CompletedToolCall } from '../core/coreToolScheduler.js';
|
||||||
|
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
|
||||||
import { FileDiff } from '../tools/tools.js';
|
import { FileDiff } from '../tools/tools.js';
|
||||||
import { AuthType } from '../core/contentGenerator.js';
|
import { AuthType } from '../core/contentGenerator.js';
|
||||||
import {
|
import {
|
||||||
|
@ -114,6 +115,7 @@ export class ToolCallEvent implements BaseTelemetryEvent {
|
||||||
error?: string;
|
error?: string;
|
||||||
error_type?: string;
|
error_type?: string;
|
||||||
prompt_id: string;
|
prompt_id: string;
|
||||||
|
tool_type: 'native' | 'mcp';
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
metadata?: { [key: string]: any };
|
metadata?: { [key: string]: any };
|
||||||
|
|
||||||
|
@ -130,6 +132,10 @@ export class ToolCallEvent implements BaseTelemetryEvent {
|
||||||
this.error = call.response.error?.message;
|
this.error = call.response.error?.message;
|
||||||
this.error_type = call.response.errorType;
|
this.error_type = call.response.errorType;
|
||||||
this.prompt_id = call.request.prompt_id;
|
this.prompt_id = call.request.prompt_id;
|
||||||
|
this.tool_type =
|
||||||
|
typeof call.tool !== 'undefined' && call.tool instanceof DiscoveredMCPTool
|
||||||
|
? 'mcp'
|
||||||
|
: 'native';
|
||||||
|
|
||||||
if (
|
if (
|
||||||
call.status === 'success' &&
|
call.status === 'success' &&
|
||||||
|
|
Loading…
Reference in New Issue