diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddc1288d..d34f39a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,14 @@ jobs: - name: Run linter run: npm run lint:ci + - name: Run linter on integration tests + run: npx eslint integration-tests --max-warnings 0 + + - name: Run formatter on integration tests + run: | + npx prettier --check integration-tests + git diff --exit-code + - name: Build project run: npm run build diff --git a/integration-tests/file-system.test.js b/integration-tests/file-system.test.ts similarity index 100% rename from integration-tests/file-system.test.js rename to integration-tests/file-system.test.ts diff --git a/integration-tests/google_web_search.test.js b/integration-tests/google_web_search.test.ts similarity index 90% rename from integration-tests/google_web_search.test.js rename to integration-tests/google_web_search.test.ts index 31747421..6fb365a0 100644 --- a/integration-tests/google_web_search.test.js +++ b/integration-tests/google_web_search.test.ts @@ -18,10 +18,13 @@ test('should be able to search the web', async () => { } catch (error) { // Network errors can occur in CI environments if ( - error.message.includes('network') || - error.message.includes('timeout') + error instanceof Error && + (error.message.includes('network') || error.message.includes('timeout')) ) { - console.warn('Skipping test due to network error:', error.message); + console.warn( + 'Skipping test due to network error:', + (error as Error).message, + ); return; // Skip the test } throw error; // Re-throw if not a network error diff --git a/integration-tests/list_directory.test.js b/integration-tests/list_directory.test.ts similarity index 94% rename from integration-tests/list_directory.test.js rename to integration-tests/list_directory.test.ts index 16f49f4b..023eca12 100644 --- a/integration-tests/list_directory.test.js +++ b/integration-tests/list_directory.test.ts @@ -21,8 +21,8 @@ test('should be able to list a directory', async () => { await rig.poll( () => { // Check if the files exist in the test directory - const file1Path = join(rig.testDir, 'file1.txt'); - const subdirPath = join(rig.testDir, 'subdir'); + const file1Path = join(rig.testDir!, 'file1.txt'); + const subdirPath = join(rig.testDir!, 'subdir'); return existsSync(file1Path) && existsSync(subdirPath); }, 1000, // 1 second max wait diff --git a/integration-tests/read_many_files.test.js b/integration-tests/read_many_files.test.ts similarity index 100% rename from integration-tests/read_many_files.test.js rename to integration-tests/read_many_files.test.ts diff --git a/integration-tests/replace.test.js b/integration-tests/replace.test.ts similarity index 100% rename from integration-tests/replace.test.js rename to integration-tests/replace.test.ts diff --git a/integration-tests/run-tests.js b/integration-tests/run-tests.js index 05fb349e..b33e1afa 100644 --- a/integration-tests/run-tests.js +++ b/integration-tests/run-tests.js @@ -52,13 +52,13 @@ async function main() { const testPatterns = args.length > 0 - ? args.map((arg) => `integration-tests/${arg}.test.js`) - : ['integration-tests/*.test.js']; + ? args.map((arg) => `integration-tests/${arg}.test.ts`) + : ['integration-tests/*.test.ts']; const testFiles = glob.sync(testPatterns, { cwd: rootDir, absolute: true }); for (const testFile of testFiles) { const testFileName = basename(testFile); - console.log(`\tFound test file: ${testFileName}`); + console.log(` Found test file: ${testFileName}`); } const MAX_RETRIES = 3; @@ -92,7 +92,7 @@ async function main() { } nodeArgs.push(testFile); - const child = spawn('node', nodeArgs, { + const child = spawn('npx', ['tsx', ...nodeArgs], { stdio: 'pipe', env: { ...process.env, diff --git a/integration-tests/run_shell_command.test.js b/integration-tests/run_shell_command.test.ts similarity index 100% rename from integration-tests/run_shell_command.test.js rename to integration-tests/run_shell_command.test.ts diff --git a/integration-tests/save_memory.test.js b/integration-tests/save_memory.test.ts similarity index 100% rename from integration-tests/save_memory.test.js rename to integration-tests/save_memory.test.ts diff --git a/integration-tests/simple-mcp-server.test.js b/integration-tests/simple-mcp-server.test.ts similarity index 96% rename from integration-tests/simple-mcp-server.test.js rename to integration-tests/simple-mcp-server.test.ts index 987f69d2..c4191078 100644 --- a/integration-tests/simple-mcp-server.test.js +++ b/integration-tests/simple-mcp-server.test.ts @@ -14,11 +14,8 @@ import { test, describe, before } from 'node:test'; import { strict as assert } from 'node:assert'; import { TestRig, validateModelOutput } from './test-helper.js'; import { join } from 'path'; -import { fileURLToPath } from 'url'; import { writeFileSync } from 'fs'; -const __dirname = fileURLToPath(new URL('.', import.meta.url)); - // Create a minimal MCP server that doesn't require external dependencies // This implements the MCP protocol directly using Node.js built-ins const serverScript = `#!/usr/bin/env node @@ -185,7 +182,7 @@ describe('simple-mcp-server', () => { }); // Create server script in the test directory - const testServerPath = join(rig.testDir, 'mcp-server.cjs'); + const testServerPath = join(rig.testDir!, 'mcp-server.cjs'); writeFileSync(testServerPath, serverScript); // Make the script executable (though running with 'node' should work anyway) diff --git a/integration-tests/test-helper.js b/integration-tests/test-helper.ts similarity index 85% rename from integration-tests/test-helper.js rename to integration-tests/test-helper.ts index d1125a78..33443aaf 100644 --- a/integration-tests/test-helper.js +++ b/integration-tests/test-helper.ts @@ -14,7 +14,7 @@ import { fileExists } from '../scripts/telemetry_utils.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); -function sanitizeTestName(name) { +function sanitizeTestName(name: string) { return name .toLowerCase() .replace(/[^a-z0-9]/g, '-') @@ -22,7 +22,11 @@ function sanitizeTestName(name) { } // Helper to create detailed error messages -export function createToolCallErrorMessage(expectedTools, foundTools, result) { +export function createToolCallErrorMessage( + expectedTools: string | string[], + foundTools: string[], + result: string, +) { const expectedStr = Array.isArray(expectedTools) ? expectedTools.join(' or ') : expectedTools; @@ -34,7 +38,11 @@ export function createToolCallErrorMessage(expectedTools, foundTools, result) { } // Helper to print debug information when tests fail -export function printDebugInfo(rig, result, context = {}) { +export function printDebugInfo( + rig: TestRig, + result: string, + context: Record = {}, +) { console.error('Test failed - Debug info:'); console.error('Result length:', result.length); console.error('Result (first 500 chars):', result.substring(0, 500)); @@ -60,8 +68,8 @@ export function printDebugInfo(rig, result, context = {}) { // Helper to validate model output and warn about unexpected content export function validateModelOutput( - result, - expectedContent = null, + result: string, + expectedContent: string | (string | RegExp)[] | null = null, testName = '', ) { // First, check if there's any output at all (this should fail the test if missing) @@ -102,6 +110,11 @@ export function validateModelOutput( } export class TestRig { + bundlePath: string; + testDir: string | null; + testName?: string; + _lastRunStdout?: string; + constructor() { this.bundlePath = join(__dirname, '..', 'bundle/gemini.js'); this.testDir = null; @@ -114,10 +127,13 @@ export class TestRig { return 15000; // 15s locally } - setup(testName, options = {}) { + setup( + testName: string, + options: { settings?: Record } = {}, + ) { this.testName = testName; const sanitizedName = sanitizeTestName(testName); - this.testDir = join(env.INTEGRATION_TEST_FILE_DIR, sanitizedName); + this.testDir = join(env.INTEGRATION_TEST_FILE_DIR!, sanitizedName); mkdirSync(this.testDir, { recursive: true }); // Create a settings file to point the CLI to the local collector @@ -146,25 +162,32 @@ export class TestRig { ); } - createFile(fileName, content) { - const filePath = join(this.testDir, fileName); + createFile(fileName: string, content: string) { + const filePath = join(this.testDir!, fileName); writeFileSync(filePath, content); return filePath; } - mkdir(dir) { - mkdirSync(join(this.testDir, dir), { recursive: true }); + mkdir(dir: string) { + mkdirSync(join(this.testDir!, dir), { recursive: true }); } sync() { // ensure file system is done before spawning - execSync('sync', { cwd: this.testDir }); + execSync('sync', { cwd: this.testDir! }); } - run(promptOrOptions, ...args) { + run( + promptOrOptions: string | { prompt?: string; stdin?: string }, + ...args: string[] + ): Promise { let command = `node ${this.bundlePath} --yolo`; - const execOptions = { - cwd: this.testDir, + const execOptions: { + cwd: string; + encoding: 'utf-8'; + input?: string; + } = { + cwd: this.testDir!, encoding: 'utf-8', }; @@ -185,10 +208,10 @@ export class TestRig { command += ` ${args.join(' ')}`; const commandArgs = parse(command); - const node = commandArgs.shift(); + const node = commandArgs.shift() as string; - const child = spawn(node, commandArgs, { - cwd: this.testDir, + const child = spawn(node, commandArgs as string[], { + cwd: this.testDir!, stdio: 'pipe', }); @@ -197,26 +220,26 @@ export class TestRig { // Handle stdin if provided if (execOptions.input) { - child.stdin.write(execOptions.input); - child.stdin.end(); + child.stdin!.write(execOptions.input); + child.stdin!.end(); } - child.stdout.on('data', (data) => { + child.stdout!.on('data', (data: Buffer) => { stdout += data; if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') { process.stdout.write(data); } }); - child.stderr.on('data', (data) => { + child.stderr!.on('data', (data: Buffer) => { stderr += data; if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') { process.stderr.write(data); } }); - const promise = new Promise((resolve, reject) => { - child.on('close', (code) => { + const promise = new Promise((resolve, reject) => { + child.on('close', (code: number) => { if (code === 0) { // Store the raw stdout for Podman telemetry parsing this._lastRunStdout = stdout; @@ -273,13 +296,13 @@ export class TestRig { return promise; } - readFile(fileName) { - const content = readFileSync(join(this.testDir, fileName), 'utf-8'); + readFile(fileName: string) { + const content = readFileSync(join(this.testDir!, fileName), 'utf-8'); if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') { - const testId = `${env.TEST_FILE_NAME.replace( + const testId = `${env.TEST_FILE_NAME!.replace( '.test.js', '', - )}:${this.testName.replace(/ /g, '-')}`; + )}:${this.testName!.replace(/ /g, '-')}`; console.log(`--- FILE: ${testId}/${fileName} ---`); console.log(content); console.log(`--- END FILE: ${testId}/${fileName} ---`); @@ -295,7 +318,7 @@ export class TestRig { } catch (error) { // Ignore cleanup errors if (env.VERBOSE === 'true') { - console.warn('Cleanup warning:', error.message); + console.warn('Cleanup warning:', (error as Error).message); } } } @@ -305,7 +328,7 @@ export class TestRig { // In sandbox mode, telemetry is written to a relative path in the test directory const logFilePath = env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false' - ? join(this.testDir, 'telemetry.log') + ? join(this.testDir!, 'telemetry.log') : env.TELEMETRY_LOG_FILE; if (!logFilePath) return; @@ -318,7 +341,7 @@ export class TestRig { const content = readFileSync(logFilePath, 'utf-8'); // Check if file has meaningful content (at least one complete JSON object) return content.includes('"event.name"'); - } catch (_e) { + } catch { return false; } }, @@ -327,7 +350,7 @@ export class TestRig { ); } - async waitForToolCall(toolName, timeout) { + async waitForToolCall(toolName: string, timeout?: number) { // Use environment-specific timeout if (!timeout) { timeout = this.getDefaultTimeout(); @@ -346,7 +369,7 @@ export class TestRig { ); } - async waitForAnyToolCall(toolNames, timeout) { + async waitForAnyToolCall(toolNames: string[], timeout?: number) { // Use environment-specific timeout if (!timeout) { timeout = this.getDefaultTimeout(); @@ -367,7 +390,11 @@ export class TestRig { ); } - async poll(predicate, timeout, interval) { + async poll( + predicate: () => boolean, + timeout: number, + interval: number, + ): Promise { const startTime = Date.now(); let attempts = 0; while (Date.now() - startTime < timeout) { @@ -389,8 +416,16 @@ export class TestRig { return false; } - _parseToolLogsFromStdout(stdout) { - const logs = []; + _parseToolLogsFromStdout(stdout: string) { + const logs: { + timestamp: number; + toolRequest: { + name: string; + args: string; + success: boolean; + duration_ms: number; + }; + }[] = []; // The console output from Podman is JavaScript object notation, not JSON // Look for tool call events in the output @@ -493,7 +528,7 @@ export class TestRig { }, }); } - } catch (_e) { + } catch { // Not valid JSON } currentObject = ''; @@ -510,7 +545,7 @@ export class TestRig { // If not, fall back to parsing from stdout if (env.GEMINI_SANDBOX === 'podman') { // Try reading from file first - const logFilePath = join(this.testDir, 'telemetry.log'); + const logFilePath = join(this.testDir!, 'telemetry.log'); if (fileExists(logFilePath)) { try { @@ -522,7 +557,7 @@ export class TestRig { // File exists but is empty or doesn't have events, parse from stdout return this._parseToolLogsFromStdout(this._lastRunStdout); } - } catch (_e) { + } catch { // Error reading file, fall back to stdout if (this._lastRunStdout) { return this._parseToolLogsFromStdout(this._lastRunStdout); @@ -537,7 +572,7 @@ export class TestRig { // In sandbox mode, telemetry is written to a relative path in the test directory const logFilePath = env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false' - ? join(this.testDir, 'telemetry.log') + ? join(this.testDir!, 'telemetry.log') : env.TELEMETRY_LOG_FILE; if (!logFilePath) { @@ -553,7 +588,7 @@ export class TestRig { const content = readFileSync(logFilePath, 'utf-8'); // Split the content into individual JSON objects - // They are separated by "}\n{" pattern + // They are separated by "}\n{" const jsonObjects = content .split(/}\s*\n\s*{/) .map((obj, index, array) => { @@ -564,7 +599,14 @@ export class TestRig { }) .filter((obj) => obj); - const logs = []; + const logs: { + toolRequest: { + name: string; + args: string; + success: boolean; + duration_ms: number; + }; + }[] = []; for (const jsonStr of jsonObjects) { try { @@ -584,10 +626,13 @@ export class TestRig { }, }); } - } catch (_e) { + } catch (e) { // Skip objects that aren't valid JSON if (env.VERBOSE === 'true') { - console.error('Failed to parse telemetry object:', _e.message); + console.error( + 'Failed to parse telemetry object:', + (e as Error).message, + ); } } } diff --git a/integration-tests/tsconfig.json b/integration-tests/tsconfig.json new file mode 100644 index 00000000..3e053d04 --- /dev/null +++ b/integration-tests/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "allowJs": true + }, + "include": ["**/*.ts"] +} diff --git a/integration-tests/write_file.test.js b/integration-tests/write_file.test.ts similarity index 100% rename from integration-tests/write_file.test.js rename to integration-tests/write_file.test.ts diff --git a/package-lock.json b/package-lock.json index 9ee6b751..dfe65ded 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "mock-fs": "^5.5.0", "prettier": "^3.5.3", "react-devtools-core": "^4.28.5", + "tsx": "^4.20.3", "typescript-eslint": "^8.30.1", "vitest": "^3.2.4", "yargs": "^17.7.2" @@ -5692,6 +5693,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", + "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -9262,6 +9276,16 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/restore-cursor": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", @@ -10562,6 +10586,26 @@ "dev": true, "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.20.3", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.3.tgz", + "integrity": "sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.25.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 0ba06955..637fc445 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "mock-fs": "^5.5.0", "prettier": "^3.5.3", "react-devtools-core": "^4.28.5", + "tsx": "^4.20.3", "typescript-eslint": "^8.30.1", "vitest": "^3.2.4", "yargs": "^17.7.2",