Address flaky integration tests with retries (#4604)

integration-tests/run-tests.js

@@ -61,6 +61,7 @@ async function main() {
     console.log(`\tFound test file: ${testFileName}`);
   }
 
+  const MAX_RETRIES = 3;
   let allTestsPassed = true;
 
   for (const testFile of testFiles) {
@@ -72,6 +73,19 @@ async function main() {
       `------------- Running test file: ${testFileName} ------------------------------`,
     );
 
+    let attempt = 0;
+    let testFilePassed = false;
+    let lastStdout = [];
+    let lastStderr = [];
+
+    while (attempt < MAX_RETRIES && !testFilePassed) {
+      attempt++;
+      if (attempt > 1) {
+        console.log(
+          `--- Retrying ${testFileName} (attempt ${attempt} of ${MAX_RETRIES}) ---`,
+        );
+      }
+
       const nodeArgs = ['--test'];
       if (verbose) {
         nodeArgs.push('--test-reporter=spec');
@@ -92,14 +106,19 @@ async function main() {
 
       let outputStream;
       if (keepOutput) {
-      const outputFile = join(testFileDir, 'output.log');
+        const outputFile = join(testFileDir, `output-attempt-${attempt}.log`);
         outputStream = createWriteStream(outputFile);
         console.log(`Output for ${testFileName} written to: ${outputFile}`);
       }
 
+      const stdout = [];
+      const stderr = [];
+
       child.stdout.on('data', (data) => {
         if (verbose) {
           process.stdout.write(data);
+        } else {
+          stdout.push(data);
         }
         if (outputStream) {
           outputStream.write(data);
@@ -109,6 +128,8 @@ async function main() {
       child.stderr.on('data', (data) => {
         if (verbose) {
           process.stderr.write(data);
+        } else {
+          stderr.push(data);
         }
         if (outputStream) {
           outputStream.write(data);
@@ -127,8 +148,22 @@ async function main() {
         });
       });
 
-    if (exitCode !== 0) {
+      if (exitCode === 0) {
-      console.error(`Test file failed: ${testFileName}`);
+        testFilePassed = true;
+      } else {
+        lastStdout = stdout;
+        lastStderr = stderr;
+      }
+    }
+
+    if (!testFilePassed) {
+      console.error(
+        `Test file failed after ${MAX_RETRIES} attempts: ${testFileName}`,
+      );
+      if (!verbose) {
+        process.stdout.write(Buffer.concat(lastStdout).toString('utf8'));
+        process.stderr.write(Buffer.concat(lastStderr).toString('utf8'));
+      }
       allTestsPassed = false;
     }
   }