jcarr
/
gemini-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improvements to web-fetch tool (#1030)

This commit is contained in:
Allen Hutchison 2025-06-13 17:44:14 -07:00 committed by GitHub
parent 8eb505fbba
commit 31b28ade01
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 5151 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/tools/web-fetch.md
View File

@ -8,9 +8,11 @@ This document describes the `web_fetch` tool.
- **Arguments:**
- `prompt` (string, required): A comprehensive prompt that includes the URL(s) (up to 20) to fetch and specific instructions on how to process their content. For example: `"Summarize https://example.com/article and extract key points from https://another.com/data"`. The prompt must contain at least one URL starting with `http://` or `https://`.
- **Behavior:**
- The tool sends the prompt and the specified URLs to the Gemini API.
- The API fetches the content of the URLs, processes it according to the instructions in the prompt, and returns a consolidated response.
- The tool formats the response, including source attribution with citations, and returns it to the user.
- The tool will first ask for confirmation before fetching any URLs.
- It attempts to process URLs through the Gemini API's `urlContext` tool first.
- If the Gemini API cannot access a URL (e.g., it's on `localhost`, a private network, or behind a firewall), the tool will fall back to fetching the content directly from the local machine.
- The tool can fetch content from `localhost` and private network addresses via this fallback mechanism.
- The tool formats the response, including source attribution with citations where possible, and returns it to the user.
- **Examples:**
- Summarizing a single article:
```

187
package-lock.json generated
View File

@ -2087,6 +2087,19 @@
"dev": true,
"license": "MIT"
},
"node_modules/@selderee/plugin-htmlparser2": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz",
"integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==",
"license": "MIT",
"dependencies": {
"domhandler": "^5.0.3",
"selderee": "^0.11.0"
},
"funding": {
"url": "https://ko-fi.com/killymxi"
}
},
"node_modules/@testing-library/dom": {
"version": "9.3.4",
"resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.4.tgz",
@ -2216,6 +2229,12 @@
"@types/unist": "*"
}
},
"node_modules/@types/html-to-text": {
"version": "9.0.4",
"resolved": "https://registry.npmjs.org/@types/html-to-text/-/html-to-text-9.0.4.tgz",
"integrity": "sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ==",
"license": "MIT"
},
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@ -3866,6 +3885,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/default-browser": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz",
@ -4014,6 +4042,73 @@
"dev": true,
"license": "MIT"
},
"node_modules/dom-serializer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"license": "MIT",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/dom-serializer/node_modules/entities": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "BSD-2-Clause"
},
"node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"license": "BSD-2-Clause",
"dependencies": {
"domelementtype": "^2.3.0"
},
"engines": {
"node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/domutils": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
"license": "BSD-2-Clause",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dotenv": {
"version": "16.5.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
@ -5540,6 +5635,53 @@
"dev": true,
"license": "MIT"
},
"node_modules/html-to-text": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz",
"integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==",
"license": "MIT",
"dependencies": {
"@selderee/plugin-htmlparser2": "^0.11.0",
"deepmerge": "^4.3.1",
"dom-serializer": "^2.0.0",
"htmlparser2": "^8.0.2",
"selderee": "^0.11.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/htmlparser2": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
"integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==",
"funding": [
"https://github.com/fb55/htmlparser2?sponsor=1",
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "MIT",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3",
"domutils": "^3.0.1",
"entities": "^4.4.0"
}
},
"node_modules/htmlparser2/node_modules/entities": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@ -6824,6 +6966,15 @@
"node": ">=0.10.0"
}
},
"node_modules/leac": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz",
"integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==",
"license": "MIT",
"funding": {
"url": "https://ko-fi.com/killymxi"
}
},
"node_modules/levn": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@ -7552,6 +7703,19 @@
"url": "https://github.com/inikulin/parse5?sponsor=1"
}
},
"node_modules/parseley": {
"version": "0.12.1",
"resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz",
"integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==",
"license": "MIT",
"dependencies": {
"leac": "^0.6.0",
"peberminta": "^0.9.0"
},
"funding": {
"url": "https://ko-fi.com/killymxi"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@ -7637,6 +7801,15 @@
"node": ">= 14.16"
}
},
"node_modules/peberminta": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz",
"integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==",
"license": "MIT",
"funding": {
"url": "https://ko-fi.com/killymxi"
}
},
"node_modules/picocolors": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
@ -8319,6 +8492,18 @@
"loose-envify": "^1.1.0"
}
},
"node_modules/selderee": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz",
"integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==",
"license": "MIT",
"dependencies": {
"parseley": "^0.12.0"
},
"funding": {
"url": "https://ko-fi.com/killymxi"
}
},
"node_modules/semver": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
@ -10811,10 +10996,12 @@
"@opentelemetry/instrumentation-http": "^0.52.0",
"@opentelemetry/sdk-node": "^0.52.0",
"@types/glob": "^8.1.0",
"@types/html-to-text": "^9.0.4",
"diff": "^7.0.0",
"dotenv": "^16.4.7",
"glob": "^10.4.5",
"google-auth-library": "^9.11.0",
"html-to-text": "^9.0.5",
"ignore": "^7.0.0",
"micromatch": "^4.0.8",
"open": "^10.1.2",

50
packages/cli/src/ui/components/messages/ToolConfirmationMessage.test.tsx Normal file
View File

@ -0,0 +1,50 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { render } from 'ink-testing-library';
import { describe, it, expect, vi } from 'vitest';
import { ToolConfirmationMessage } from './ToolConfirmationMessage.js';
import { ToolCallConfirmationDetails } from '@gemini-cli/core';
describe('ToolConfirmationMessage', () => {
it('should not display urls if prompt and url are the same', () => {
const confirmationDetails: ToolCallConfirmationDetails = {
type: 'info',
title: 'Confirm Web Fetch',
prompt: 'https://example.com',
urls: ['https://example.com'],
onConfirm: vi.fn(),
};
const { lastFrame } = render(
<ToolConfirmationMessage confirmationDetails={confirmationDetails} />,
);
expect(lastFrame()).not.toContain('URLs to fetch:');
});
it('should display urls if prompt and url are different', () => {
const confirmationDetails: ToolCallConfirmationDetails = {
type: 'info',
title: 'Confirm Web Fetch',
prompt:
'fetch https://github.com/google/gemini-react/blob/main/README.md',
urls: [
'https://raw.githubusercontent.com/google/gemini-react/main/README.md',
],
onConfirm: vi.fn(),
};
const { lastFrame } = render(
<ToolConfirmationMessage confirmationDetails={confirmationDetails} />,
);
expect(lastFrame()).toContain('URLs to fetch:');
expect(lastFrame()).toContain(
'- https://raw.githubusercontent.com/google/gemini-react/main/README.md',
);
});
});

32
packages/cli/src/ui/components/messages/ToolConfirmationMessage.tsx
View File

@ -115,6 +115,38 @@ export const ToolConfirmationMessage: React.FC<
},
{ label: 'No (esc)', value: ToolConfirmationOutcome.Cancel },
);
} else if (confirmationDetails.type === 'info') {
const infoProps = confirmationDetails;
const displayUrls =
infoProps.urls &&
!(infoProps.urls.length === 1 && infoProps.urls[0] === infoProps.prompt);
bodyContent = (
<Box flexDirection="column" paddingX={1} marginLeft={1}>
<Text color={Colors.AccentCyan}>{infoProps.prompt}</Text>
{displayUrls && infoProps.urls && infoProps.urls.length > 0 && (
<Box flexDirection="column" marginTop={1}>
<Text>URLs to fetch:</Text>
{infoProps.urls.map((url) => (
<Text key={url}> - {url}</Text>
))}
</Box>
)}
</Box>
);
question = `Do you want to proceed?`;
options.push(
{
label: 'Yes, allow once',
value: ToolConfirmationOutcome.ProceedOnce,
},
{
label: 'Yes, allow always',
value: ToolConfirmationOutcome.ProceedAlways,
},
{ label: 'No (esc)', value: ToolConfirmationOutcome.Cancel },
);
} else {
// mcp tool confirmation
const mcpProps = confirmationDetails as ToolMcpConfirmationDetails;

4583
packages/core/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

2
packages/core/package.json
View File

@ -30,10 +30,12 @@
"@opentelemetry/instrumentation-http": "^0.52.0",
"@opentelemetry/sdk-node": "^0.52.0",
"@types/glob": "^8.1.0",
"@types/html-to-text": "^9.0.4",
"diff": "^7.0.0",
"dotenv": "^16.4.7",
"glob": "^10.4.5",
"google-auth-library": "^9.11.0",
"html-to-text": "^9.0.5",
"ignore": "^7.0.0",
"micromatch": "^4.0.8",
"open": "^10.1.2",

11
packages/core/src/tools/tools.ts
View File

@ -222,10 +222,19 @@ export interface ToolMcpConfirmationDetails {
onConfirm: (outcome: ToolConfirmationOutcome) => Promise<void>;
}
export interface ToolInfoConfirmationDetails {
type: 'info';
title: string;
onConfirm: (outcome: ToolConfirmationOutcome) => Promise<void>;
prompt: string;
urls?: string[];
}
export type ToolCallConfirmationDetails =
| ToolEditConfirmationDetails
| ToolExecuteConfirmationDetails
| ToolMcpConfirmationDetails;
| ToolMcpConfirmationDetails
| ToolInfoConfirmationDetails;
export enum ToolConfirmationOutcome {
ProceedOnce = 'proceed_once',

86
packages/core/src/tools/web-fetch.test.ts Normal file
View File

@ -0,0 +1,86 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi } from 'vitest';
import { WebFetchTool } from './web-fetch.js';
import { Config, ApprovalMode } from '../config/config.js';
import { ToolConfirmationOutcome } from './tools.js';
describe('WebFetchTool', () => {
const mockConfig = {
getApprovalMode: vi.fn(),
setApprovalMode: vi.fn(),
} as unknown as Config;
describe('shouldConfirmExecute', () => {
it('should return confirmation details with the correct prompt and urls', async () => {
const tool = new WebFetchTool(mockConfig);
const params = { prompt: 'fetch https://example.com' };
const confirmationDetails = await tool.shouldConfirmExecute(params);
expect(confirmationDetails).toEqual({
type: 'info',
title: 'Confirm Web Fetch',
prompt: 'fetch https://example.com',
urls: ['https://example.com'],
onConfirm: expect.any(Function),
});
});
it('should convert github urls to raw format', async () => {
const tool = new WebFetchTool(mockConfig);
const params = {
prompt:
'fetch https://github.com/google/gemini-react/blob/main/README.md',
};
const confirmationDetails = await tool.shouldConfirmExecute(params);
expect(confirmationDetails).toEqual({
type: 'info',
title: 'Confirm Web Fetch',
prompt:
'fetch https://github.com/google/gemini-react/blob/main/README.md',
urls: [
'https://raw.githubusercontent.com/google/gemini-react/main/README.md',
],
onConfirm: expect.any(Function),
});
});
it('should return false if approval mode is AUTO_EDIT', async () => {
const tool = new WebFetchTool({
...mockConfig,
getApprovalMode: () => ApprovalMode.AUTO_EDIT,
} as unknown as Config);
const params = { prompt: 'fetch https://example.com' };
const confirmationDetails = await tool.shouldConfirmExecute(params);
expect(confirmationDetails).toBe(false);
});
it('should call setApprovalMode when onConfirm is called with ProceedAlways', async () => {
const setApprovalMode = vi.fn();
const tool = new WebFetchTool({
...mockConfig,
setApprovalMode,
} as unknown as Config);
const params = { prompt: 'fetch https://example.com' };
const confirmationDetails = await tool.shouldConfirmExecute(params);
if (
confirmationDetails &&
typeof confirmationDetails === 'object' &&
'onConfirm' in confirmationDetails
) {
await confirmationDetails.onConfirm(
ToolConfirmationOutcome.ProceedAlways,
);
}
expect(setApprovalMode).toHaveBeenCalledWith(ApprovalMode.AUTO_EDIT);
});
});
});

154
packages/core/src/tools/web-fetch.ts
View File

@ -6,10 +6,26 @@
import { GroundingMetadata } from '@google/genai';
import { SchemaValidator } from '../utils/schemaValidator.js';
import { BaseTool, ToolResult } from './tools.js';
import {
BaseTool,
ToolResult,
ToolCallConfirmationDetails,
ToolConfirmationOutcome,
} from './tools.js';
import { getErrorMessage } from '../utils/errors.js';
import { Config } from '../config/config.js';
import { Config, ApprovalMode } from '../config/config.js';
import { getResponseText } from '../utils/generateContentResponseUtilities.js';
import { fetchWithTimeout, isPrivateIp } from '../utils/fetch.js';
import { convert } from 'html-to-text';
const URL_FETCH_TIMEOUT_MS = 10000;
const MAX_CONTENT_LENGTH = 100000;
// Helper function to extract URLs from a string
function extractUrls(text: string): string[] {
const urlRegex = /(https?:\/\/[^\s]+)/g;
return text.match(urlRegex) || [];
}
// Interfaces for grounding metadata (similar to web-search.ts)
interface GroundingChunkWeb {
@ -52,7 +68,7 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
super(
WebFetchTool.Name,
'WebFetch',
"Processes content from URL(s) embedded in a prompt. Include up to 20 URLs and instructions (e.g., summarize, extract specific data) directly in the 'prompt' parameter.",
"Processes content from URL(s), including local and private network addresses (e.g., localhost), embedded in a prompt. Include up to 20 URLs and instructions (e.g., summarize, extract specific data) directly in the 'prompt' parameter.",
{
properties: {
prompt: {
@ -67,6 +83,71 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
);
}
private async executeFallback(
params: WebFetchToolParams,
signal: AbortSignal,
): Promise<ToolResult> {
const urls = extractUrls(params.prompt);
if (urls.length === 0) {
return {
llmContent: 'Error: No URL found in the prompt for fallback.',
returnDisplay: 'Error: No URL found in the prompt for fallback.',
};
}
// For now, we only support one URL for fallback
let url = urls[0];
// Convert GitHub blob URL to raw URL
if (url.includes('github.com') && url.includes('/blob/')) {
url = url
.replace('github.com', 'raw.githubusercontent.com')
.replace('/blob/', '/');
}
try {
const response = await fetchWithTimeout(url, URL_FETCH_TIMEOUT_MS);
if (!response.ok) {
throw new Error(
`Request failed with status code ${response.status} ${response.statusText}`,
);
}
const html = await response.text();
const textContent = convert(html, {
wordwrap: false,
selectors: [
{ selector: 'a', options: { ignoreHref: true } },
{ selector: 'img', format: 'skip' },
],
}).substring(0, MAX_CONTENT_LENGTH);
const geminiClient = this.config.getGeminiClient();
const fallbackPrompt = `The user requested the following: "${params.prompt}".
I was unable to access the URL directly. Instead, I have fetched the raw content of the page. Please use the following content to answer the user's request. Do not attempt to access the URL again.
---
${textContent}
---`;
const result = await geminiClient.generateContent(
[{ role: 'user', parts: [{ text: fallbackPrompt }] }],
{},
signal,
);
const resultText = getResponseText(result) || '';
return {
llmContent: resultText,
returnDisplay: `Content for ${url} processed using fallback fetch.`,
};
} catch (e) {
const error = e as Error;
const errorMessage = `Error during fallback fetch for ${url}: ${error.message}`;
return {
llmContent: `Error: ${errorMessage}`,
returnDisplay: `Error: ${errorMessage}`,
};
}
}
validateParams(params: WebFetchToolParams): string | null {
if (
this.schema.parameters &&
@ -97,6 +178,43 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
return `Processing URLs and instructions from prompt: "${displayPrompt}"`;
}
async shouldConfirmExecute(
params: WebFetchToolParams,
): Promise<ToolCallConfirmationDetails | false> {
if (this.config.getApprovalMode() === ApprovalMode.AUTO_EDIT) {
return false;
}
const validationError = this.validateParams(params);
if (validationError) {
return false;
}
// Perform GitHub URL conversion here to differentiate between user-provided
// URL and the actual URL to be fetched.
const urls = extractUrls(params.prompt).map((url) => {
if (url.includes('github.com') && url.includes('/blob/')) {
return url
.replace('github.com', 'raw.githubusercontent.com')
.replace('/blob/', '/');
}
return url;
});
const confirmationDetails: ToolCallConfirmationDetails = {
type: 'info',
title: `Confirm Web Fetch`,
prompt: params.prompt,
urls,
onConfirm: async (outcome: ToolConfirmationOutcome) => {
if (outcome === ToolConfirmationOutcome.ProceedAlways) {
this.config.setApprovalMode(ApprovalMode.AUTO_EDIT);
}
},
};
return confirmationDetails;
}
async execute(
params: WebFetchToolParams,
signal: AbortSignal,
@ -110,6 +228,14 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
}
const userPrompt = params.prompt;
const urls = extractUrls(userPrompt);
const url = urls[0];
const isPrivate = isPrivateIp(url);
if (isPrivate) {
return this.executeFallback(params, signal);
}
const geminiClient = this.config.getGeminiClient();
try {
@ -120,7 +246,10 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
);
console.debug(
`[WebFetchTool] Full response for prompt "${userPrompt.substring(0, 50)}...":`,
`[WebFetchTool] Full response for prompt "${userPrompt.substring(
0,
50,
)}...":`,
JSON.stringify(response, null, 2),
);
@ -138,7 +267,6 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
// Error Handling
let processingError = false;
let errorDetail = 'An unknown error occurred during content processing.';
if (
urlContextMeta?.urlMetadata &&
@ -149,13 +277,10 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
);
if (allStatuses.every((s) => s !== 'URL_RETRIEVAL_STATUS_SUCCESS')) {
processingError = true;
errorDetail = `All URL retrieval attempts failed. Statuses: ${allStatuses.join(', ')}. API reported: "${responseText || 'No additional detail.'}"`;
}
} else if (!responseText.trim() && !sources?.length) {
// No URL metadata and no content/sources
processingError = true;
errorDetail =
'No content was returned and no URL metadata was available to determine fetch status.';
}
if (
@ -165,16 +290,10 @@ export class WebFetchTool extends BaseTool<WebFetchToolParams, ToolResult> {
) {
// Successfully retrieved some URL (or no specific error from urlContextMeta), but no usable text or grounding data.
processingError = true;
errorDetail =
'URL(s) processed, but no substantive content or grounding information was found.';
}
if (processingError) {
const errorText = `Failed to process prompt and fetch URL data. ${errorDetail}`;
return {
llmContent: `Error: ${errorText}`,
returnDisplay: `Error: ${errorText}`,
};
return this.executeFallback(params, signal);
}
const sourceListFormatted: string[] = [];
@ -227,7 +346,10 @@ ${sourceListFormatted.join('\n')}`;
returnDisplay: `Content processed from prompt.`,
};
} catch (error: unknown) {
const errorMessage = `Error processing web content for prompt "${userPrompt.substring(0, 50)}...": ${getErrorMessage(error)}`;
const errorMessage = `Error processing web content for prompt "${userPrompt.substring(
0,
50,
)}...": ${getErrorMessage(error)}`;
console.error(errorMessage, error);
return {
llmContent: `Error: ${errorMessage}`,

57
packages/core/src/utils/fetch.ts Normal file
View File

@ -0,0 +1,57 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { getErrorMessage, isNodeError } from './errors.js';
import { URL } from 'url';
const PRIVATE_IP_RANGES = [
/^10\./,
/^127\./,
/^172\.(1[6-9]|2[0-9]|3[0-1])\./,
/^192\.168\./,
/^::1$/,
/^fc00:/,
/^fe80:/,
];
export class FetchError extends Error {
constructor(
message: string,
public code?: string,
) {
super(message);
this.name = 'FetchError';
}
}
export function isPrivateIp(url: string): boolean {
try {
const hostname = new URL(url).hostname;
return PRIVATE_IP_RANGES.some((range) => range.test(hostname));
} catch (_e) {
return false;
}
}
export async function fetchWithTimeout(
url: string,
timeout: number,
): Promise<Response> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const response = await fetch(url, { signal: controller.signal });
return response;
} catch (error) {
if (isNodeError(error) && error.code === 'ABORT_ERR') {
throw new FetchError(`Request timed out after ${timeout}ms`, 'ETIMEDOUT');
}
throw new FetchError(getErrorMessage(error));
} finally {
clearTimeout(timeoutId);
}
}

2
packages/core/src/utils/getFolderStructure.test.ts
View File

@ -38,8 +38,8 @@ const createDirent = (name: string, type: 'file' | 'dir'): FSDirent => ({
isSymbolicLink: () => false,
isFIFO: () => false,
isSocket: () => false,
parentPath: '',
path: '',
parentPath: '',
});
describe('getFolderStructure', () => {