Add integration test for maximum schema depth error handling (#5685)

This commit is contained in:
Jacob MacDonald 2025-08-06 13:45:54 -07:00 committed by GitHub
parent b3cfaeb6d3
commit e3e7677753
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 227 additions and 17 deletions

View File

@ -28,6 +28,7 @@ export default tseslint.config(
// Global ignores
ignores: [
'node_modules/*',
'.integration-tests/**',
'eslint.config.js',
'packages/cli/dist/**',
'packages/core/dist/**',

View File

@ -0,0 +1,206 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* This test verifies we can match maximum schema depth errors from Gemini
* and then detect and warn about the potential tools that caused the error.
*/
import { test, describe, before } from 'node:test';
import { strict as assert } from 'node:assert';
import { TestRig } from './test-helper.js';
import { join } from 'path';
import { fileURLToPath } from 'url';
import { writeFileSync, readFileSync } from 'fs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
// Create a minimal MCP server that doesn't require external dependencies
// This implements the MCP protocol directly using Node.js built-ins
const serverScript = `#!/usr/bin/env node
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
const readline = require('readline');
const fs = require('fs');
// Debug logging to stderr (only when MCP_DEBUG or VERBOSE is set)
const debugEnabled = process.env.MCP_DEBUG === 'true' || process.env.VERBOSE === 'true';
function debug(msg) {
if (debugEnabled) {
fs.writeSync(2, \`[MCP-DEBUG] \${msg}\\n\`);
}
}
debug('MCP server starting...');
// Simple JSON-RPC implementation for MCP
class SimpleJSONRPC {
constructor() {
this.handlers = new Map();
this.rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false
});
this.rl.on('line', (line) => {
debug(\`Received line: \${line}\`);
try {
const message = JSON.parse(line);
debug(\`Parsed message: \${JSON.stringify(message)}\`);
this.handleMessage(message);
} catch (e) {
debug(\`Parse error: \${e.message}\`);
}
});
}
send(message) {
const msgStr = JSON.stringify(message);
debug(\`Sending message: \${msgStr}\`);
process.stdout.write(msgStr + '\\n');
}
async handleMessage(message) {
if (message.method && this.handlers.has(message.method)) {
try {
const result = await this.handlers.get(message.method)(message.params || {});
if (message.id !== undefined) {
this.send({
jsonrpc: '2.0',
id: message.id,
result
});
}
} catch (error) {
if (message.id !== undefined) {
this.send({
jsonrpc: '2.0',
id: message.id,
error: {
code: -32603,
message: error.message
}
});
}
}
} else if (message.id !== undefined) {
this.send({
jsonrpc: '2.0',
id: message.id,
error: {
code: -32601,
message: 'Method not found'
}
});
}
}
on(method, handler) {
this.handlers.set(method, handler);
}
}
// Create MCP server
const rpc = new SimpleJSONRPC();
// Handle initialize
rpc.on('initialize', async (params) => {
debug('Handling initialize request');
return {
protocolVersion: '2024-11-05',
capabilities: {
tools: {}
},
serverInfo: {
name: 'cyclic-schema-server',
version: '1.0.0'
}
};
});
// Handle tools/list
rpc.on('tools/list', async () => {
debug('Handling tools/list request');
return {
tools: [{
name: 'tool_with_cyclic_schema',
inputSchema: {
type: 'object',
properties: {
data: {
type: 'array',
items: {
type: 'object',
properties: {
child: { $ref: '#/properties/data/items' },
},
},
},
},
}
}]
};
});
// Send initialization notification
rpc.send({
jsonrpc: '2.0',
method: 'initialized'
});
`;
describe('mcp server with cyclic tool schema is detected', () => {
const rig = new TestRig();
before(async () => {
// Setup test directory with MCP server configuration
await rig.setup('cyclic-schema-mcp-server', {
settings: {
mcpServers: {
'cyclic-schema-server': {
command: 'node',
args: ['mcp-server.cjs'],
},
},
},
});
// Create server script in the test directory
const testServerPath = join(rig.testDir, 'mcp-server.cjs');
writeFileSync(testServerPath, serverScript);
// Make the script executable (though running with 'node' should work anyway)
if (process.platform !== 'win32') {
const { chmodSync } = await import('fs');
chmodSync(testServerPath, 0o755);
}
});
test('should error and suggest disabling the cyclic tool', async () => {
// Just run any command to trigger the schema depth error.
// If this test starts failing, check `isSchemaDepthError` from
// geminiChat.ts to see if it needs to be updated.
// Or, possibly it could mean that gemini has fixed the issue.
const output = await rig.run('hello');
// The error message is in a log file, so we need to extract the path and read it.
const match = output.match(/Full report available at: (.*\.json)/);
assert(match, `Could not find log file path in output: ${output}`);
const logFilePath = match[1];
const logFileContent = readFileSync(logFilePath, 'utf-8');
assert.match(
logFileContent,
/ - tool_with_cyclic_schema \(cyclic-schema-server MCP Server\)/,
);
});
});

View File

@ -258,6 +258,11 @@ export class TestRig {
result = filteredLines.join('\n');
}
// If we have stderr output, include that also
if (stderr) {
result += `\n\nStdErr:\n${stderr}`;
}
resolve(result);
} else {
reject(new Error(`Process exited with code ${code}:\n${stderr}`));

View File

@ -300,16 +300,14 @@ export class GeminiChat {
};
response = await retryWithBackoff(apiCall, {
shouldRetry: (error: Error) => {
// Check for likely cyclic schema errors, don't retry those.
if (error.message.includes('maximum schema depth exceeded'))
return false;
// Check error messages for status codes, or specific error names if known
if (error && error.message) {
shouldRetry: (error: unknown) => {
// Check for known error messages and codes.
if (error instanceof Error && error.message) {
if (isSchemaDepthError(error.message)) return false;
if (error.message.includes('429')) return true;
if (error.message.match(/5\d{2}/)) return true;
}
return false;
return false; // Don't retry other errors by default
},
onPersistent429: async (authType?: string, error?: unknown) =>
await this.handleFlashFallback(authType, error),
@ -419,12 +417,10 @@ export class GeminiChat {
// the stream. For simple 429/500 errors on initial call, this is fine.
// If errors occur mid-stream, this setup won't resume the stream; it will restart it.
const streamResponse = await retryWithBackoff(apiCall, {
shouldRetry: (error: Error) => {
// Check for likely cyclic schema errors, don't retry those.
if (error.message.includes('maximum schema depth exceeded'))
return false;
// Check error messages for status codes, or specific error names if known
if (error && error.message) {
shouldRetry: (error: unknown) => {
// Check for known error messages and codes.
if (error instanceof Error && error.message) {
if (isSchemaDepthError(error.message)) return false;
if (error.message.includes('429')) return true;
if (error.message.match(/5\d{2}/)) return true;
}
@ -689,10 +685,7 @@ export class GeminiChat {
private async maybeIncludeSchemaDepthContext(error: unknown): Promise<void> {
// Check for potentially problematic cyclic tools with cyclic schemas
// and include a recommendation to remove potentially problematic tools.
if (
isStructuredError(error) &&
error.message.includes('maximum schema depth exceeded')
) {
if (isStructuredError(error) && isSchemaDepthError(error.message)) {
const tools = (await this.config.getToolRegistry()).getAllTools();
const cyclicSchemaTools: string[] = [];
for (const tool of tools) {
@ -714,3 +707,8 @@ export class GeminiChat {
}
}
}
/** Visible for Testing */
export function isSchemaDepthError(errorMessage: string): boolean {
return errorMessage.includes('maximum schema depth exceeded');
}