Add integration test for maximum schema depth error handling (#5685)
This commit is contained in:
parent
b3cfaeb6d3
commit
e3e7677753
|
@ -28,6 +28,7 @@ export default tseslint.config(
|
|||
// Global ignores
|
||||
ignores: [
|
||||
'node_modules/*',
|
||||
'.integration-tests/**',
|
||||
'eslint.config.js',
|
||||
'packages/cli/dist/**',
|
||||
'packages/core/dist/**',
|
||||
|
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* This test verifies we can match maximum schema depth errors from Gemini
|
||||
* and then detect and warn about the potential tools that caused the error.
|
||||
*/
|
||||
|
||||
import { test, describe, before } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { TestRig } from './test-helper.js';
|
||||
import { join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { writeFileSync, readFileSync } from 'fs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
|
||||
// Create a minimal MCP server that doesn't require external dependencies
|
||||
// This implements the MCP protocol directly using Node.js built-ins
|
||||
const serverScript = `#!/usr/bin/env node
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
const readline = require('readline');
|
||||
const fs = require('fs');
|
||||
|
||||
// Debug logging to stderr (only when MCP_DEBUG or VERBOSE is set)
|
||||
const debugEnabled = process.env.MCP_DEBUG === 'true' || process.env.VERBOSE === 'true';
|
||||
function debug(msg) {
|
||||
if (debugEnabled) {
|
||||
fs.writeSync(2, \`[MCP-DEBUG] \${msg}\\n\`);
|
||||
}
|
||||
}
|
||||
|
||||
debug('MCP server starting...');
|
||||
|
||||
// Simple JSON-RPC implementation for MCP
|
||||
class SimpleJSONRPC {
|
||||
constructor() {
|
||||
this.handlers = new Map();
|
||||
this.rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
terminal: false
|
||||
});
|
||||
|
||||
this.rl.on('line', (line) => {
|
||||
debug(\`Received line: \${line}\`);
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
debug(\`Parsed message: \${JSON.stringify(message)}\`);
|
||||
this.handleMessage(message);
|
||||
} catch (e) {
|
||||
debug(\`Parse error: \${e.message}\`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
send(message) {
|
||||
const msgStr = JSON.stringify(message);
|
||||
debug(\`Sending message: \${msgStr}\`);
|
||||
process.stdout.write(msgStr + '\\n');
|
||||
}
|
||||
|
||||
async handleMessage(message) {
|
||||
if (message.method && this.handlers.has(message.method)) {
|
||||
try {
|
||||
const result = await this.handlers.get(message.method)(message.params || {});
|
||||
if (message.id !== undefined) {
|
||||
this.send({
|
||||
jsonrpc: '2.0',
|
||||
id: message.id,
|
||||
result
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (message.id !== undefined) {
|
||||
this.send({
|
||||
jsonrpc: '2.0',
|
||||
id: message.id,
|
||||
error: {
|
||||
code: -32603,
|
||||
message: error.message
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (message.id !== undefined) {
|
||||
this.send({
|
||||
jsonrpc: '2.0',
|
||||
id: message.id,
|
||||
error: {
|
||||
code: -32601,
|
||||
message: 'Method not found'
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
on(method, handler) {
|
||||
this.handlers.set(method, handler);
|
||||
}
|
||||
}
|
||||
|
||||
// Create MCP server
|
||||
const rpc = new SimpleJSONRPC();
|
||||
|
||||
// Handle initialize
|
||||
rpc.on('initialize', async (params) => {
|
||||
debug('Handling initialize request');
|
||||
return {
|
||||
protocolVersion: '2024-11-05',
|
||||
capabilities: {
|
||||
tools: {}
|
||||
},
|
||||
serverInfo: {
|
||||
name: 'cyclic-schema-server',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Handle tools/list
|
||||
rpc.on('tools/list', async () => {
|
||||
debug('Handling tools/list request');
|
||||
return {
|
||||
tools: [{
|
||||
name: 'tool_with_cyclic_schema',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
data: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
child: { $ref: '#/properties/data/items' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}]
|
||||
};
|
||||
});
|
||||
|
||||
// Send initialization notification
|
||||
rpc.send({
|
||||
jsonrpc: '2.0',
|
||||
method: 'initialized'
|
||||
});
|
||||
`;
|
||||
|
||||
describe('mcp server with cyclic tool schema is detected', () => {
|
||||
const rig = new TestRig();
|
||||
|
||||
before(async () => {
|
||||
// Setup test directory with MCP server configuration
|
||||
await rig.setup('cyclic-schema-mcp-server', {
|
||||
settings: {
|
||||
mcpServers: {
|
||||
'cyclic-schema-server': {
|
||||
command: 'node',
|
||||
args: ['mcp-server.cjs'],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Create server script in the test directory
|
||||
const testServerPath = join(rig.testDir, 'mcp-server.cjs');
|
||||
writeFileSync(testServerPath, serverScript);
|
||||
|
||||
// Make the script executable (though running with 'node' should work anyway)
|
||||
if (process.platform !== 'win32') {
|
||||
const { chmodSync } = await import('fs');
|
||||
chmodSync(testServerPath, 0o755);
|
||||
}
|
||||
});
|
||||
|
||||
test('should error and suggest disabling the cyclic tool', async () => {
|
||||
// Just run any command to trigger the schema depth error.
|
||||
// If this test starts failing, check `isSchemaDepthError` from
|
||||
// geminiChat.ts to see if it needs to be updated.
|
||||
// Or, possibly it could mean that gemini has fixed the issue.
|
||||
const output = await rig.run('hello');
|
||||
|
||||
// The error message is in a log file, so we need to extract the path and read it.
|
||||
const match = output.match(/Full report available at: (.*\.json)/);
|
||||
assert(match, `Could not find log file path in output: ${output}`);
|
||||
|
||||
const logFilePath = match[1];
|
||||
const logFileContent = readFileSync(logFilePath, 'utf-8');
|
||||
|
||||
assert.match(
|
||||
logFileContent,
|
||||
/ - tool_with_cyclic_schema \(cyclic-schema-server MCP Server\)/,
|
||||
);
|
||||
});
|
||||
});
|
|
@ -258,6 +258,11 @@ export class TestRig {
|
|||
|
||||
result = filteredLines.join('\n');
|
||||
}
|
||||
// If we have stderr output, include that also
|
||||
if (stderr) {
|
||||
result += `\n\nStdErr:\n${stderr}`;
|
||||
}
|
||||
|
||||
resolve(result);
|
||||
} else {
|
||||
reject(new Error(`Process exited with code ${code}:\n${stderr}`));
|
||||
|
|
|
@ -300,16 +300,14 @@ export class GeminiChat {
|
|||
};
|
||||
|
||||
response = await retryWithBackoff(apiCall, {
|
||||
shouldRetry: (error: Error) => {
|
||||
// Check for likely cyclic schema errors, don't retry those.
|
||||
if (error.message.includes('maximum schema depth exceeded'))
|
||||
return false;
|
||||
// Check error messages for status codes, or specific error names if known
|
||||
if (error && error.message) {
|
||||
shouldRetry: (error: unknown) => {
|
||||
// Check for known error messages and codes.
|
||||
if (error instanceof Error && error.message) {
|
||||
if (isSchemaDepthError(error.message)) return false;
|
||||
if (error.message.includes('429')) return true;
|
||||
if (error.message.match(/5\d{2}/)) return true;
|
||||
}
|
||||
return false;
|
||||
return false; // Don't retry other errors by default
|
||||
},
|
||||
onPersistent429: async (authType?: string, error?: unknown) =>
|
||||
await this.handleFlashFallback(authType, error),
|
||||
|
@ -419,12 +417,10 @@ export class GeminiChat {
|
|||
// the stream. For simple 429/500 errors on initial call, this is fine.
|
||||
// If errors occur mid-stream, this setup won't resume the stream; it will restart it.
|
||||
const streamResponse = await retryWithBackoff(apiCall, {
|
||||
shouldRetry: (error: Error) => {
|
||||
// Check for likely cyclic schema errors, don't retry those.
|
||||
if (error.message.includes('maximum schema depth exceeded'))
|
||||
return false;
|
||||
// Check error messages for status codes, or specific error names if known
|
||||
if (error && error.message) {
|
||||
shouldRetry: (error: unknown) => {
|
||||
// Check for known error messages and codes.
|
||||
if (error instanceof Error && error.message) {
|
||||
if (isSchemaDepthError(error.message)) return false;
|
||||
if (error.message.includes('429')) return true;
|
||||
if (error.message.match(/5\d{2}/)) return true;
|
||||
}
|
||||
|
@ -689,10 +685,7 @@ export class GeminiChat {
|
|||
private async maybeIncludeSchemaDepthContext(error: unknown): Promise<void> {
|
||||
// Check for potentially problematic cyclic tools with cyclic schemas
|
||||
// and include a recommendation to remove potentially problematic tools.
|
||||
if (
|
||||
isStructuredError(error) &&
|
||||
error.message.includes('maximum schema depth exceeded')
|
||||
) {
|
||||
if (isStructuredError(error) && isSchemaDepthError(error.message)) {
|
||||
const tools = (await this.config.getToolRegistry()).getAllTools();
|
||||
const cyclicSchemaTools: string[] = [];
|
||||
for (const tool of tools) {
|
||||
|
@ -714,3 +707,8 @@ export class GeminiChat {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Visible for Testing */
|
||||
export function isSchemaDepthError(errorMessage: string): boolean {
|
||||
return errorMessage.includes('maximum schema depth exceeded');
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue