chore(integration-tests): refactor to typescript (#5645)

.github/workflows/ci.yml

@@ -36,6 +36,14 @@ jobs:
       - name: Run linter
         run: npm run lint:ci
 
+      - name: Run linter on integration tests
+        run: npx eslint integration-tests --max-warnings 0
+
+      - name: Run formatter on integration tests
+        run: |
+          npx prettier --check integration-tests
+          git diff --exit-code
+
       - name: Build project
         run: npm run build
 

integration-tests/google_web_search.test.ts

@@ -18,10 +18,13 @@ test('should be able to search the web', async () => {
   } catch (error) {
     // Network errors can occur in CI environments
     if (
-      error.message.includes('network') ||
-      error.message.includes('timeout')
+      error instanceof Error &&
+      (error.message.includes('network') || error.message.includes('timeout'))
     ) {
-      console.warn('Skipping test due to network error:', error.message);
+      console.warn(
+        'Skipping test due to network error:',
+        (error as Error).message,
+      );
       return; // Skip the test
     }
     throw error; // Re-throw if not a network error

integration-tests/list_directory.test.ts

@@ -21,8 +21,8 @@ test('should be able to list a directory', async () => {
   await rig.poll(
     () => {
       // Check if the files exist in the test directory
-      const file1Path = join(rig.testDir, 'file1.txt');
-      const subdirPath = join(rig.testDir, 'subdir');
+      const file1Path = join(rig.testDir!, 'file1.txt');
+      const subdirPath = join(rig.testDir!, 'subdir');
       return existsSync(file1Path) && existsSync(subdirPath);
     },
     1000, // 1 second max wait

integration-tests/run-tests.js

@@ -52,13 +52,13 @@ async function main() {
 
   const testPatterns =
     args.length > 0
-      ? args.map((arg) => `integration-tests/${arg}.test.js`)
-      : ['integration-tests/*.test.js'];
+      ? args.map((arg) => `integration-tests/${arg}.test.ts`)
+      : ['integration-tests/*.test.ts'];
   const testFiles = glob.sync(testPatterns, { cwd: rootDir, absolute: true });
 
   for (const testFile of testFiles) {
     const testFileName = basename(testFile);
-    console.log(`\tFound test file: ${testFileName}`);
+    console.log(`	Found test file: ${testFileName}`);
   }
 
   const MAX_RETRIES = 3;
@@ -92,7 +92,7 @@ async function main() {
       }
       nodeArgs.push(testFile);
 
-      const child = spawn('node', nodeArgs, {
+      const child = spawn('npx', ['tsx', ...nodeArgs], {
         stdio: 'pipe',
         env: {
           ...process.env,

integration-tests/simple-mcp-server.test.ts

@@ -14,11 +14,8 @@ import { test, describe, before } from 'node:test';
 import { strict as assert } from 'node:assert';
 import { TestRig, validateModelOutput } from './test-helper.js';
 import { join } from 'path';
-import { fileURLToPath } from 'url';
 import { writeFileSync } from 'fs';
 
-const __dirname = fileURLToPath(new URL('.', import.meta.url));
-
 // Create a minimal MCP server that doesn't require external dependencies
 // This implements the MCP protocol directly using Node.js built-ins
 const serverScript = `#!/usr/bin/env node
@@ -185,7 +182,7 @@ describe('simple-mcp-server', () => {
     });
 
     // Create server script in the test directory
-    const testServerPath = join(rig.testDir, 'mcp-server.cjs');
+    const testServerPath = join(rig.testDir!, 'mcp-server.cjs');
     writeFileSync(testServerPath, serverScript);
 
     // Make the script executable (though running with 'node' should work anyway)

integration-tests/test-helper.ts

@@ -14,7 +14,7 @@ import { fileExists } from '../scripts/telemetry_utils.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-function sanitizeTestName(name) {
+function sanitizeTestName(name: string) {
   return name
     .toLowerCase()
     .replace(/[^a-z0-9]/g, '-')
@@ -22,7 +22,11 @@ function sanitizeTestName(name) {
 }
 
 // Helper to create detailed error messages
-export function createToolCallErrorMessage(expectedTools, foundTools, result) {
+export function createToolCallErrorMessage(
+  expectedTools: string | string[],
+  foundTools: string[],
+  result: string,
+) {
   const expectedStr = Array.isArray(expectedTools)
     ? expectedTools.join(' or ')
     : expectedTools;
@@ -34,7 +38,11 @@ export function createToolCallErrorMessage(expectedTools, foundTools, result) {
 }
 
 // Helper to print debug information when tests fail
-export function printDebugInfo(rig, result, context = {}) {
+export function printDebugInfo(
+  rig: TestRig,
+  result: string,
+  context: Record<string, unknown> = {},
+) {
   console.error('Test failed - Debug info:');
   console.error('Result length:', result.length);
   console.error('Result (first 500 chars):', result.substring(0, 500));
@@ -60,8 +68,8 @@ export function printDebugInfo(rig, result, context = {}) {
 
 // Helper to validate model output and warn about unexpected content
 export function validateModelOutput(
-  result,
-  expectedContent = null,
+  result: string,
+  expectedContent: string | (string | RegExp)[] | null = null,
   testName = '',
 ) {
   // First, check if there's any output at all (this should fail the test if missing)
@@ -102,6 +110,11 @@ export function validateModelOutput(
 }
 
 export class TestRig {
+  bundlePath: string;
+  testDir: string | null;
+  testName?: string;
+  _lastRunStdout?: string;
+
   constructor() {
     this.bundlePath = join(__dirname, '..', 'bundle/gemini.js');
     this.testDir = null;
@@ -114,10 +127,13 @@ export class TestRig {
     return 15000; // 15s locally
   }
 
-  setup(testName, options = {}) {
+  setup(
+    testName: string,
+    options: { settings?: Record<string, unknown> } = {},
+  ) {
     this.testName = testName;
     const sanitizedName = sanitizeTestName(testName);
-    this.testDir = join(env.INTEGRATION_TEST_FILE_DIR, sanitizedName);
+    this.testDir = join(env.INTEGRATION_TEST_FILE_DIR!, sanitizedName);
     mkdirSync(this.testDir, { recursive: true });
 
     // Create a settings file to point the CLI to the local collector
@@ -146,25 +162,32 @@ export class TestRig {
     );
   }
 
-  createFile(fileName, content) {
-    const filePath = join(this.testDir, fileName);
+  createFile(fileName: string, content: string) {
+    const filePath = join(this.testDir!, fileName);
     writeFileSync(filePath, content);
     return filePath;
   }
 
-  mkdir(dir) {
-    mkdirSync(join(this.testDir, dir), { recursive: true });
+  mkdir(dir: string) {
+    mkdirSync(join(this.testDir!, dir), { recursive: true });
   }
 
   sync() {
     // ensure file system is done before spawning
-    execSync('sync', { cwd: this.testDir });
+    execSync('sync', { cwd: this.testDir! });
   }
 
-  run(promptOrOptions, ...args) {
+  run(
+    promptOrOptions: string | { prompt?: string; stdin?: string },
+    ...args: string[]
+  ): Promise<string> {
     let command = `node ${this.bundlePath} --yolo`;
-    const execOptions = {
-      cwd: this.testDir,
+    const execOptions: {
+      cwd: string;
+      encoding: 'utf-8';
+      input?: string;
+    } = {
+      cwd: this.testDir!,
       encoding: 'utf-8',
     };
 
@@ -185,10 +208,10 @@ export class TestRig {
     command += ` ${args.join(' ')}`;
 
     const commandArgs = parse(command);
-    const node = commandArgs.shift();
+    const node = commandArgs.shift() as string;
 
-    const child = spawn(node, commandArgs, {
-      cwd: this.testDir,
+    const child = spawn(node, commandArgs as string[], {
+      cwd: this.testDir!,
       stdio: 'pipe',
     });
 
@@ -197,26 +220,26 @@ export class TestRig {
 
     // Handle stdin if provided
     if (execOptions.input) {
-      child.stdin.write(execOptions.input);
-      child.stdin.end();
+      child.stdin!.write(execOptions.input);
+      child.stdin!.end();
     }
 
-    child.stdout.on('data', (data) => {
+    child.stdout!.on('data', (data: Buffer) => {
       stdout += data;
       if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
         process.stdout.write(data);
       }
     });
 
-    child.stderr.on('data', (data) => {
+    child.stderr!.on('data', (data: Buffer) => {
       stderr += data;
       if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
         process.stderr.write(data);
       }
     });
 
-    const promise = new Promise((resolve, reject) => {
-      child.on('close', (code) => {
+    const promise = new Promise<string>((resolve, reject) => {
+      child.on('close', (code: number) => {
         if (code === 0) {
           // Store the raw stdout for Podman telemetry parsing
           this._lastRunStdout = stdout;
@@ -273,13 +296,13 @@ export class TestRig {
     return promise;
   }
 
-  readFile(fileName) {
-    const content = readFileSync(join(this.testDir, fileName), 'utf-8');
+  readFile(fileName: string) {
+    const content = readFileSync(join(this.testDir!, fileName), 'utf-8');
     if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
-      const testId = `${env.TEST_FILE_NAME.replace(
+      const testId = `${env.TEST_FILE_NAME!.replace(
         '.test.js',
         '',
-      )}:${this.testName.replace(/ /g, '-')}`;
+      )}:${this.testName!.replace(/ /g, '-')}`;
       console.log(`--- FILE: ${testId}/${fileName} ---`);
       console.log(content);
       console.log(`--- END FILE: ${testId}/${fileName} ---`);
@@ -295,7 +318,7 @@ export class TestRig {
       } catch (error) {
         // Ignore cleanup errors
         if (env.VERBOSE === 'true') {
-          console.warn('Cleanup warning:', error.message);
+          console.warn('Cleanup warning:', (error as Error).message);
         }
       }
     }
@@ -305,7 +328,7 @@ export class TestRig {
     // In sandbox mode, telemetry is written to a relative path in the test directory
     const logFilePath =
       env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false'
-        ? join(this.testDir, 'telemetry.log')
+        ? join(this.testDir!, 'telemetry.log')
         : env.TELEMETRY_LOG_FILE;
 
     if (!logFilePath) return;
@@ -318,7 +341,7 @@ export class TestRig {
           const content = readFileSync(logFilePath, 'utf-8');
           // Check if file has meaningful content (at least one complete JSON object)
           return content.includes('"event.name"');
-        } catch (_e) {
+        } catch {
           return false;
         }
       },
@@ -327,7 +350,7 @@ export class TestRig {
     );
   }
 
-  async waitForToolCall(toolName, timeout) {
+  async waitForToolCall(toolName: string, timeout?: number) {
     // Use environment-specific timeout
     if (!timeout) {
       timeout = this.getDefaultTimeout();
@@ -346,7 +369,7 @@ export class TestRig {
     );
   }
 
-  async waitForAnyToolCall(toolNames, timeout) {
+  async waitForAnyToolCall(toolNames: string[], timeout?: number) {
     // Use environment-specific timeout
     if (!timeout) {
       timeout = this.getDefaultTimeout();
@@ -367,7 +390,11 @@ export class TestRig {
     );
   }
 
-  async poll(predicate, timeout, interval) {
+  async poll(
+    predicate: () => boolean,
+    timeout: number,
+    interval: number,
+  ): Promise<boolean> {
     const startTime = Date.now();
     let attempts = 0;
     while (Date.now() - startTime < timeout) {
@@ -389,8 +416,16 @@ export class TestRig {
     return false;
   }
 
-  _parseToolLogsFromStdout(stdout) {
-    const logs = [];
+  _parseToolLogsFromStdout(stdout: string) {
+    const logs: {
+      timestamp: number;
+      toolRequest: {
+        name: string;
+        args: string;
+        success: boolean;
+        duration_ms: number;
+      };
+    }[] = [];
 
     // The console output from Podman is JavaScript object notation, not JSON
     // Look for tool call events in the output
@@ -493,7 +528,7 @@ export class TestRig {
                   },
                 });
               }
-            } catch (_e) {
+            } catch {
               // Not valid JSON
             }
             currentObject = '';
@@ -510,7 +545,7 @@ export class TestRig {
     // If not, fall back to parsing from stdout
     if (env.GEMINI_SANDBOX === 'podman') {
       // Try reading from file first
-      const logFilePath = join(this.testDir, 'telemetry.log');
+      const logFilePath = join(this.testDir!, 'telemetry.log');
 
       if (fileExists(logFilePath)) {
         try {
@@ -522,7 +557,7 @@ export class TestRig {
             // File exists but is empty or doesn't have events, parse from stdout
             return this._parseToolLogsFromStdout(this._lastRunStdout);
           }
-        } catch (_e) {
+        } catch {
           // Error reading file, fall back to stdout
           if (this._lastRunStdout) {
             return this._parseToolLogsFromStdout(this._lastRunStdout);
@@ -537,7 +572,7 @@ export class TestRig {
     // In sandbox mode, telemetry is written to a relative path in the test directory
     const logFilePath =
       env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false'
-        ? join(this.testDir, 'telemetry.log')
+        ? join(this.testDir!, 'telemetry.log')
         : env.TELEMETRY_LOG_FILE;
 
     if (!logFilePath) {
@@ -553,7 +588,7 @@ export class TestRig {
     const content = readFileSync(logFilePath, 'utf-8');
 
     // Split the content into individual JSON objects
-    // They are separated by "}\n{" pattern
+    // They are separated by "}\n{"
     const jsonObjects = content
       .split(/}\s*\n\s*{/)
       .map((obj, index, array) => {
@@ -564,7 +599,14 @@ export class TestRig {
       })
       .filter((obj) => obj);
 
-    const logs = [];
+    const logs: {
+      toolRequest: {
+        name: string;
+        args: string;
+        success: boolean;
+        duration_ms: number;
+      };
+    }[] = [];
 
     for (const jsonStr of jsonObjects) {
       try {
@@ -584,10 +626,13 @@ export class TestRig {
             },
           });
         }
-      } catch (_e) {
+      } catch (e) {
         // Skip objects that aren't valid JSON
         if (env.VERBOSE === 'true') {
-          console.error('Failed to parse telemetry object:', _e.message);
+          console.error(
+            'Failed to parse telemetry object:',
+            (e as Error).message,
+          );
         }
       }
     }

integration-tests/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "allowJs": true
+  },
+  "include": ["**/*.ts"]
+}

package-lock.json

@@ -39,6 +39,7 @@
         "mock-fs": "^5.5.0",
         "prettier": "^3.5.3",
         "react-devtools-core": "^4.28.5",
+        "tsx": "^4.20.3",
         "typescript-eslint": "^8.30.1",
         "vitest": "^3.2.4",
         "yargs": "^17.7.2"
@@ -5692,6 +5693,19 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/get-tsconfig": {
+      "version": "4.10.1",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz",
+      "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "resolve-pkg-maps": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
+      }
+    },
     "node_modules/glob": {
       "version": "10.4.5",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@@ -9262,6 +9276,16 @@
         "node": ">=4"
       }
     },
+    "node_modules/resolve-pkg-maps": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
+      "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
+      "dev": true,
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
+      }
+    },
     "node_modules/restore-cursor": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz",
@@ -10562,6 +10586,26 @@
       "dev": true,
       "license": "0BSD"
     },
+    "node_modules/tsx": {
+      "version": "4.20.3",
+      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.3.tgz",
+      "integrity": "sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "esbuild": "~0.25.0",
+        "get-tsconfig": "^4.7.5"
+      },
+      "bin": {
+        "tsx": "dist/cli.mjs"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.3"
+      }
+    },
     "node_modules/type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",

package.json

@@ -82,6 +82,7 @@
     "mock-fs": "^5.5.0",
     "prettier": "^3.5.3",
     "react-devtools-core": "^4.28.5",
+    "tsx": "^4.20.3",
     "typescript-eslint": "^8.30.1",
     "vitest": "^3.2.4",
     "yargs": "^17.7.2",