jcarr
/
gemini-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix Telemetry for tool calls, PR 1/n] Propagate tool reported errors via ToolCallResponseInfo and ToolResult (#5222)

This commit is contained in:
Silvio Junior 2025-08-01 11:20:08 -04:00 committed by GitHub
parent e126d2fcd9
commit 7748e56153
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 224 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/cli/src/nonInteractiveCli.test.ts
View File

@ -8,6 +8,7 @@ import {
Config,
executeToolCall,
ToolRegistry,
ToolErrorType,
shutdownTelemetry,
GeminiEventType,
ServerGeminiStreamEvent,
@ -161,6 +162,7 @@ describe('runNonInteractive', () => {
};
mockCoreExecuteToolCall.mockResolvedValue({
error: new Error('Tool execution failed badly'),
errorType: ToolErrorType.UNHANDLED_EXCEPTION,
});
mockGeminiClient.sendMessageStream.mockReturnValue(
createStreamFromEvents([toolCallEvent]),

7
packages/cli/src/nonInteractiveCli.ts
View File

@ -12,6 +12,7 @@ import {
shutdownTelemetry,
isTelemetrySdkInitialized,
GeminiEventType,
ToolErrorType,
} from '@google/gemini-cli-core';
import { Content, Part, FunctionCall } from '@google/genai';
@ -97,15 +98,11 @@ export async function runNonInteractive(
);
if (toolResponse.error) {
const isToolNotFound = toolResponse.error.message.includes(
'not found in registry',
);
console.error(
`Error executing tool ${fc.name}: ${toolResponse.resultDisplay || toolResponse.error.message}`,
);
if (!isToolNotFound) {
if (toolResponse.errorType === ToolErrorType.UNHANDLED_EXCEPTION)
process.exit(1);
}
}
if (toolResponse.responseParts) {

44
packages/core/src/core/coreToolScheduler.ts
View File

@ -19,6 +19,7 @@ import {
logToolCall,
ToolCallEvent,
ToolConfirmationPayload,
ToolErrorType,
} from '../index.js';
import { Part, PartListUnion } from '@google/genai';
import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js';
@ -201,6 +202,7 @@ export function convertToFunctionResponse(
const createErrorResponse = (
request: ToolCallRequestInfo,
error: Error,
errorType: ToolErrorType | undefined,
): ToolCallResponseInfo => ({
callId: request.callId,
error,
@ -212,6 +214,7 @@ const createErrorResponse = (
},
},
resultDisplay: error.message,
errorType,
});
interface CoreToolSchedulerOptions {
@ -366,6 +369,7 @@ export class CoreToolScheduler {
},
resultDisplay,
error: undefined,
errorType: undefined,
},
durationMs,
outcome,
@ -436,6 +440,7 @@ export class CoreToolScheduler {
response: createErrorResponse(
reqInfo,
new Error(`Tool "${reqInfo.name}" not found in registry.`),
ToolErrorType.TOOL_NOT_REGISTERED,
),
durationMs: 0,
};
@ -499,6 +504,7 @@ export class CoreToolScheduler {
createErrorResponse(
reqInfo,
error instanceof Error ? error : new Error(String(error)),
ToolErrorType.UNHANDLED_EXCEPTION,
),
);
}
@ -670,19 +676,30 @@ export class CoreToolScheduler {
return;
}
const response = convertToFunctionResponse(
toolName,
callId,
toolResult.llmContent,
);
const successResponse: ToolCallResponseInfo = {
callId,
responseParts: response,
resultDisplay: toolResult.returnDisplay,
error: undefined,
};
this.setStatusInternal(callId, 'success', successResponse);
if (toolResult.error === undefined) {
const response = convertToFunctionResponse(
toolName,
callId,
toolResult.llmContent,
);
const successResponse: ToolCallResponseInfo = {
callId,
responseParts: response,
resultDisplay: toolResult.returnDisplay,
error: undefined,
errorType: undefined,
};
this.setStatusInternal(callId, 'success', successResponse);
} else {
// It is a failure
const error = new Error(toolResult.error.message);
const errorResponse = createErrorResponse(
scheduledCall.request,
error,
toolResult.error.type,
);
this.setStatusInternal(callId, 'error', errorResponse);
}
})
.catch((executionError: Error) => {
this.setStatusInternal(
@ -693,6 +710,7 @@ export class CoreToolScheduler {
executionError instanceof Error
? executionError
: new Error(String(executionError)),
ToolErrorType.UNHANDLED_EXCEPTION,
),
);
});

17
packages/core/src/core/nonInteractiveToolExecutor.ts
View File

@ -8,6 +8,7 @@ import {
logToolCall,
ToolCallRequestInfo,
ToolCallResponseInfo,
ToolErrorType,
ToolRegistry,
ToolResult,
} from '../index.js';
@ -56,6 +57,7 @@ export async function executeToolCall(
],
resultDisplay: error.message,
error,
errorType: ToolErrorType.TOOL_NOT_REGISTERED,
};
}
@ -79,7 +81,11 @@ export async function executeToolCall(
function_name: toolCallRequest.name,
function_args: toolCallRequest.args,
duration_ms: durationMs,
success: true,
success: toolResult.error === undefined,
error:
toolResult.error === undefined ? undefined : toolResult.error.message,
error_type:
toolResult.error === undefined ? undefined : toolResult.error.type,
prompt_id: toolCallRequest.prompt_id,
});
@ -93,7 +99,12 @@ export async function executeToolCall(
callId: toolCallRequest.callId,
responseParts: response,
resultDisplay: tool_display,
error: undefined,
error:
toolResult.error === undefined
? undefined
: new Error(toolResult.error.message),
errorType:
toolResult.error === undefined ? undefined : toolResult.error.type,
};
} catch (e) {
const error = e instanceof Error ? e : new Error(String(e));
@ -106,6 +117,7 @@ export async function executeToolCall(
duration_ms: durationMs,
success: false,
error: error.message,
error_type: ToolErrorType.UNHANDLED_EXCEPTION,
prompt_id: toolCallRequest.prompt_id,
});
return {
@ -121,6 +133,7 @@ export async function executeToolCall(
],
resultDisplay: error.message,
error,
errorType: ToolErrorType.UNHANDLED_EXCEPTION,
};
}
}

2
packages/core/src/core/turn.ts
View File

@ -16,6 +16,7 @@ import {
ToolResult,
ToolResultDisplay,
} from '../tools/tools.js';
import { ToolErrorType } from '../tools/tool-error.js';
import { getResponseText } from '../utils/generateContentResponseUtilities.js';
import { reportError } from '../utils/errorReporting.js';
import {
@ -76,6 +77,7 @@ export interface ToolCallResponseInfo {
responseParts: PartListUnion;
resultDisplay: ToolResultDisplay | undefined;
error: Error | undefined;
errorType: ToolErrorType | undefined;
}
export interface ServerToolCallConfirmationDetails {

1
packages/core/src/index.ts
View File

@ -56,6 +56,7 @@ export * from './services/shellExecutionService.js';
// Export base tool definitions
export * from './tools/tools.js';
export * from './tools/tool-error.js';
export * from './tools/tool-registry.js';
// Export prompt logic

2
packages/core/src/telemetry/loggers.test.circular.ts
View File

@ -53,6 +53,7 @@ describe('Circular Reference Handling', () => {
responseParts: [{ text: 'test result' }],
resultDisplay: undefined,
error: undefined, // undefined means success
errorType: undefined,
};
const mockCompletedToolCall: CompletedToolCall = {
@ -100,6 +101,7 @@ describe('Circular Reference Handling', () => {
responseParts: [{ text: 'test result' }],
resultDisplay: undefined,
error: undefined, // undefined means success
errorType: undefined,
};
const mockCompletedToolCall: CompletedToolCall = {

10
packages/core/src/telemetry/loggers.test.ts
View File

@ -12,6 +12,7 @@ import {
ErroredToolCall,
GeminiClient,
ToolConfirmationOutcome,
ToolErrorType,
ToolRegistry,
} from '../index.js';
import { logs } from '@opentelemetry/api-logs';
@ -448,6 +449,7 @@ describe('loggers', () => {
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
errorType: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
@ -511,6 +513,7 @@ describe('loggers', () => {
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
errorType: undefined,
},
durationMs: 100,
outcome: ToolConfirmationOutcome.Cancel,
@ -574,6 +577,7 @@ describe('loggers', () => {
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
errorType: undefined,
},
outcome: ToolConfirmationOutcome.ModifyWithEditor,
tool: new EditTool(mockConfig),
@ -638,6 +642,7 @@ describe('loggers', () => {
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
errorType: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
@ -703,6 +708,7 @@ describe('loggers', () => {
name: 'test-error-type',
message: 'test-error',
},
errorType: ToolErrorType.UNKNOWN,
},
durationMs: 100,
};
@ -729,8 +735,8 @@ describe('loggers', () => {
success: false,
error: 'test-error',
'error.message': 'test-error',
error_type: 'test-error-type',
'error.type': 'test-error-type',
error_type: ToolErrorType.UNKNOWN,
'error.type': ToolErrorType.UNKNOWN,
prompt_id: 'prompt-id-5',
},
});

2
packages/core/src/telemetry/types.ts
View File

@ -137,7 +137,7 @@ export class ToolCallEvent {
? getDecisionFromOutcome(call.outcome)
: undefined;
this.error = call.response.error?.message;
this.error_type = call.response.error?.name;
this.error_type = call.response.errorType;
this.prompt_id = call.request.prompt_id;
}
}

3
packages/core/src/telemetry/uiTelemetry.test.ts
View File

@ -22,6 +22,7 @@ import {
ErroredToolCall,
SuccessfulToolCall,
} from '../core/coreToolScheduler.js';
import { ToolErrorType } from '../tools/tool-error.js';
import { Tool, ToolConfirmationOutcome } from '../tools/tools.js';
const createFakeCompletedToolCall = (
@ -54,6 +55,7 @@ const createFakeCompletedToolCall = (
},
},
error: undefined,
errorType: undefined,
resultDisplay: 'Success!',
},
durationMs: duration,
@ -73,6 +75,7 @@ const createFakeCompletedToolCall = (
},
},
error: error || new Error('Tool failed'),
errorType: ToolErrorType.UNKNOWN,
resultDisplay: 'Failure!',
},
durationMs: duration,

93
packages/core/src/tools/edit.test.ts
View File

@ -27,6 +27,7 @@ vi.mock('../utils/editor.js', () => ({
import { describe, it, expect, beforeEach, afterEach, vi, Mock } from 'vitest';
import { EditTool, EditToolParams } from './edit.js';
import { FileDiff } from './tools.js';
import { ToolErrorType } from './tool-error.js';
import path from 'path';
import fs from 'fs';
import os from 'os';
@ -627,6 +628,98 @@ describe('EditTool', () => {
});
});
describe('Error Scenarios', () => {
const testFile = 'error_test.txt';
let filePath: string;
beforeEach(() => {
filePath = path.join(rootDir, testFile);
});
it('should return FILE_NOT_FOUND error', async () => {
const params: EditToolParams = {
file_path: filePath,
old_string: 'any',
new_string: 'new',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(ToolErrorType.FILE_NOT_FOUND);
});
it('should return ATTEMPT_TO_CREATE_EXISTING_FILE error', async () => {
fs.writeFileSync(filePath, 'existing content', 'utf8');
const params: EditToolParams = {
file_path: filePath,
old_string: '',
new_string: 'new content',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(
ToolErrorType.ATTEMPT_TO_CREATE_EXISTING_FILE,
);
});
it('should return NO_OCCURRENCE_FOUND error', async () => {
fs.writeFileSync(filePath, 'content', 'utf8');
const params: EditToolParams = {
file_path: filePath,
old_string: 'not-found',
new_string: 'new',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(ToolErrorType.EDIT_NO_OCCURRENCE_FOUND);
});
it('should return EXPECTED_OCCURRENCE_MISMATCH error', async () => {
fs.writeFileSync(filePath, 'one one two', 'utf8');
const params: EditToolParams = {
file_path: filePath,
old_string: 'one',
new_string: 'new',
expected_replacements: 3,
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(
ToolErrorType.EDIT_EXPECTED_OCCURRENCE_MISMATCH,
);
});
it('should return NO_CHANGE error', async () => {
fs.writeFileSync(filePath, 'content', 'utf8');
const params: EditToolParams = {
file_path: filePath,
old_string: 'content',
new_string: 'content',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(ToolErrorType.EDIT_NO_CHANGE);
});
it('should return INVALID_PARAMETERS error for relative path', async () => {
const params: EditToolParams = {
file_path: 'relative/path.txt',
old_string: 'a',
new_string: 'b',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(ToolErrorType.INVALID_TOOL_PARAMS);
});
it('should return FILE_WRITE_FAILURE on write error', async () => {
fs.writeFileSync(filePath, 'content', 'utf8');
// Make file readonly to trigger a write error
fs.chmodSync(filePath, '444');
const params: EditToolParams = {
file_path: filePath,
old_string: 'content',
new_string: 'new content',
};
const result = await tool.execute(params, new AbortController().signal);
expect(result.error?.type).toBe(ToolErrorType.FILE_WRITE_FAILURE);
});
});
describe('getDescription', () => {
it('should return "No file changes to..." if old_string and new_string are the same', () => {
const testFileName = 'test.txt';

29
packages/core/src/tools/edit.ts
View File

@ -17,6 +17,7 @@ import {
ToolResult,
ToolResultDisplay,
} from './tools.js';
import { ToolErrorType } from './tool-error.js';
import { Type } from '@google/genai';
import { SchemaValidator } from '../utils/schemaValidator.js';
import { makeRelative, shortenPath } from '../utils/paths.js';
@ -62,7 +63,7 @@ interface CalculatedEdit {
currentContent: string | null;
newContent: string;
occurrences: number;
error?: { display: string; raw: string };
error?: { display: string; raw: string; type: ToolErrorType };
isNewFile: boolean;
}
@ -191,7 +192,9 @@ Expectation for required parameters:
let finalNewString = params.new_string;
let finalOldString = params.old_string;
let occurrences = 0;
let error: { display: string; raw: string } | undefined = undefined;
let error:
| { display: string; raw: string; type: ToolErrorType }
| undefined = undefined;
try {
currentContent = fs.readFileSync(params.file_path, 'utf8');
@ -214,6 +217,7 @@ Expectation for required parameters:
error = {
display: `File not found. Cannot apply edit. Use an empty old_string to create a new file.`,
raw: `File not found: ${params.file_path}`,
type: ToolErrorType.FILE_NOT_FOUND,
};
} else if (currentContent !== null) {
// Editing an existing file
@ -233,11 +237,13 @@ Expectation for required parameters:
error = {
display: `Failed to edit. Attempted to create a file that already exists.`,
raw: `File already exists, cannot create: ${params.file_path}`,
type: ToolErrorType.ATTEMPT_TO_CREATE_EXISTING_FILE,
};
} else if (occurrences === 0) {
error = {
display: `Failed to edit, could not find the string to replace.`,
raw: `Failed to edit, 0 occurrences found for old_string in ${params.file_path}. No edits made. The exact text in old_string was not found. Ensure you're not escaping content incorrectly and check whitespace, indentation, and context. Use ${ReadFileTool.Name} tool to verify.`,
type: ToolErrorType.EDIT_NO_OCCURRENCE_FOUND,
};
} else if (occurrences !== expectedReplacements) {
const occurrenceTerm =
@ -246,11 +252,13 @@ Expectation for required parameters:
error = {
display: `Failed to edit, expected ${expectedReplacements} ${occurrenceTerm} but found ${occurrences}.`,
raw: `Failed to edit, Expected ${expectedReplacements} ${occurrenceTerm} but found ${occurrences} for old_string in file: ${params.file_path}`,
type: ToolErrorType.EDIT_EXPECTED_OCCURRENCE_MISMATCH,
};
} else if (finalOldString === finalNewString) {
error = {
display: `No changes to apply. The old_string and new_string are identical.`,
raw: `No changes to apply. The old_string and new_string are identical in file: ${params.file_path}`,
type: ToolErrorType.EDIT_NO_CHANGE,
};
}
} else {
@ -258,6 +266,7 @@ Expectation for required parameters:
error = {
display: `Failed to read content of file.`,
raw: `Failed to read content of existing file: ${params.file_path}`,
type: ToolErrorType.READ_CONTENT_FAILURE,
};
}
@ -374,6 +383,10 @@ Expectation for required parameters:
return {
llmContent: `Error: Invalid parameters provided. Reason: ${validationError}`,
returnDisplay: `Error: ${validationError}`,
error: {
message: validationError,
type: ToolErrorType.INVALID_TOOL_PARAMS,
},
};
}
@ -385,6 +398,10 @@ Expectation for required parameters:
return {
llmContent: `Error preparing edit: ${errorMsg}`,
returnDisplay: `Error preparing edit: ${errorMsg}`,
error: {
message: errorMsg,
type: ToolErrorType.EDIT_PREPARATION_FAILURE,
},
};
}
@ -392,6 +409,10 @@ Expectation for required parameters:
return {
llmContent: editData.error.raw,
returnDisplay: `Error: ${editData.error.display}`,
error: {
message: editData.error.raw,
type: editData.error.type,
},
};
}
@ -442,6 +463,10 @@ Expectation for required parameters:
return {
llmContent: `Error executing edit: ${errorMsg}`,
returnDisplay: `Error writing file: ${errorMsg}`,
error: {
message: errorMsg,
type: ToolErrorType.FILE_WRITE_FAILURE,
},
};
}
}

28
packages/core/src/tools/tool-error.ts Normal file
View File

@ -0,0 +1,28 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* A type-safe enum for tool-related errors.
*/
export enum ToolErrorType {
// General Errors
INVALID_TOOL_PARAMS = 'invalid_tool_params',
UNKNOWN = 'unknown',
UNHANDLED_EXCEPTION = 'unhandled_exception',
TOOL_NOT_REGISTERED = 'tool_not_registered',
// File System Errors
FILE_NOT_FOUND = 'file_not_found',
FILE_WRITE_FAILURE = 'file_write_failure',
READ_CONTENT_FAILURE = 'read_content_failure',
ATTEMPT_TO_CREATE_EXISTING_FILE = 'attempt_to_create_existing_file',
// Edit-specific Errors
EDIT_PREPARATION_FAILURE = 'edit_preparation_failure',
EDIT_NO_OCCURRENCE_FOUND = 'edit_no_occurrence_found',
EDIT_EXPECTED_OCCURRENCE_MISMATCH = 'edit_expected_occurrence_mismatch',
EDIT_NO_CHANGE = 'edit_no_change',
}

9
packages/core/src/tools/tools.ts
View File

@ -5,6 +5,7 @@
*/
import { FunctionDeclaration, PartListUnion, Schema } from '@google/genai';
import { ToolErrorType } from './tool-error.js';
/**
* Interface representing the base Tool functionality
@ -217,6 +218,14 @@ export interface ToolResult {
* For now, we keep it as the core logic in ReadFileTool currently produces it.
*/
returnDisplay: ToolResultDisplay;
/**
* If this property is present, the tool call is considered a failure.
*/
error?: {
message: string; // raw error message
type?: ToolErrorType; // An optional machine-readable error type (e.g., 'FILE_NOT_FOUND').
};
}
export type ToolResultDisplay = string | FileDiff;