From f240fb4bea61484854cecbe711ac20d77c31915d Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Wed, 16 Nov 2022 14:40:10 +0100
Subject: [PATCH] Fix broadcasting in parallel views

---
 server/initializers/constants.ts                     |  8 --------
 .../job-queue/handlers/activitypub-http-broadcast.ts |  4 ++--
 server/lib/worker/parent-process.ts                  | 12 +++++++-----
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7039ab457..eb54781b6 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -786,14 +786,6 @@ const WORKER_THREADS = {
   PROCESS_IMAGE: {
     CONCURRENCY: 1,
     MAX_THREADS: 5
-  },
-  SEQUENTIAL_HTTP_BROADCAST: {
-    CONCURRENCY: 1,
-    MAX_THREADS: 1
-  },
-  PARALLEL_HTTP_BROADCAST: {
-    CONCURRENCY: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'],
-    MAX_THREADS: 1
   }
 }
 
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 733c1378a..57ecf0acc 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -1,7 +1,7 @@
 import { Job } from 'bullmq'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send'
 import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache'
-import { sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process'
+import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process'
 import { ActivitypubHttpBroadcastPayload } from '@shared/models'
 import { logger } from '../../../helpers/logger'
 
@@ -22,7 +22,7 @@ async function processActivityPubParallelHttpBroadcast (job: Job<ActivitypubHttp
 
   const requestOptions = await buildRequestOptions(job.data)
 
-  const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
+  const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
 
   return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
 }
diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts
index 7d4102047..48b6c682b 100644
--- a/server/lib/worker/parent-process.ts
+++ b/server/lib/worker/parent-process.ts
@@ -1,7 +1,7 @@
 import { join } from 'path'
 import Piscina from 'piscina'
 import { processImage } from '@server/helpers/image-utils'
-import { WORKER_THREADS } from '@server/initializers/constants'
+import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants'
 import { httpBroadcast } from './workers/http-broadcast'
 import { downloadImage } from './workers/image-downloader'
 
@@ -43,8 +43,9 @@ function parallelHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadca
   if (!parallelHTTPBroadcastWorker) {
     parallelHTTPBroadcastWorker = new Piscina({
       filename: join(__dirname, 'workers', 'http-broadcast.js'),
-      concurrentTasksPerWorker: WORKER_THREADS.PARALLEL_HTTP_BROADCAST.CONCURRENCY,
-      maxThreads: WORKER_THREADS.PARALLEL_HTTP_BROADCAST.MAX_THREADS
+      // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
+      concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'],
+      maxThreads: 1
     })
   }
 
@@ -59,8 +60,9 @@ function sequentialHTTPBroadcastFromWorker (options: Parameters<typeof httpBroad
   if (!sequentialHTTPBroadcastWorker) {
     sequentialHTTPBroadcastWorker = new Piscina({
       filename: join(__dirname, 'workers', 'http-broadcast.js'),
-      concurrentTasksPerWorker: WORKER_THREADS.SEQUENTIAL_HTTP_BROADCAST.CONCURRENCY,
-      maxThreads: WORKER_THREADS.SEQUENTIAL_HTTP_BROADCAST.MAX_THREADS
+      // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
+      concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'],
+      maxThreads: 1
     })
   }