2022-06-24 09:31:32 -05:00
|
|
|
import { join } from 'path'
|
2024-06-21 07:32:25 -05:00
|
|
|
import { Piscina } from 'piscina'
|
2023-07-31 07:34:36 -05:00
|
|
|
import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'
|
2023-10-19 07:18:22 -05:00
|
|
|
import type httpBroadcast from './workers/http-broadcast.js'
|
|
|
|
import type downloadImage from './workers/image-downloader.js'
|
|
|
|
import type processImage from './workers/image-processor.js'
|
|
|
|
import type getImageSize from './workers/get-image-size.js'
|
2023-10-24 03:45:17 -05:00
|
|
|
import type signJsonLDObject from './workers/sign-json-ld-object.js'
|
|
|
|
import type buildDigest from './workers/build-digest.js'
|
2023-10-25 02:43:35 -05:00
|
|
|
import type httpUnicast from './workers/http-unicast.js'
|
2023-11-21 10:29:20 -06:00
|
|
|
import { logger } from '@server/helpers/logger.js'
|
2022-06-24 09:31:32 -05:00
|
|
|
|
2022-06-28 02:09:03 -05:00
|
|
|
let downloadImageWorker: Piscina
|
2022-06-24 09:31:32 -05:00
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
export function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {
|
2022-06-28 02:09:03 -05:00
|
|
|
if (!downloadImageWorker) {
|
|
|
|
downloadImageWorker = new Piscina({
|
2023-07-31 07:34:36 -05:00
|
|
|
filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href,
|
2022-06-28 02:09:03 -05:00
|
|
|
concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS,
|
|
|
|
minThreads: 1
|
2022-06-28 02:09:03 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
downloadImageWorker.on('error', err => logger.error('Error in download image worker', { err }))
|
2022-06-28 02:09:03 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return downloadImageWorker.run(options)
|
2022-06-24 09:31:32 -05:00
|
|
|
}
|
|
|
|
|
2022-06-27 04:53:12 -05:00
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2022-06-28 02:09:03 -05:00
|
|
|
let processImageWorker: Piscina
|
2022-06-27 04:53:12 -05:00
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
export function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> {
|
2022-06-28 02:09:03 -05:00
|
|
|
if (!processImageWorker) {
|
|
|
|
processImageWorker = new Piscina({
|
2023-07-31 07:34:36 -05:00
|
|
|
filename: new URL(join('workers', 'image-processor.js'), import.meta.url).href,
|
2022-06-28 02:09:03 -05:00
|
|
|
concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY,
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS,
|
|
|
|
minThreads: 1
|
2022-06-28 02:09:03 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
processImageWorker.on('error', err => logger.error('Error in process image worker', { err }))
|
2022-06-28 02:09:03 -05:00
|
|
|
}
|
|
|
|
|
2022-06-27 04:53:12 -05:00
|
|
|
return processImageWorker.run(options)
|
|
|
|
}
|
|
|
|
|
2022-09-08 05:26:46 -05:00
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2023-10-19 07:18:22 -05:00
|
|
|
let getImageSizeWorker: Piscina
|
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
export function getImageSizeFromWorker (options: Parameters<typeof getImageSize>[0]): Promise<ReturnType<typeof getImageSize>> {
|
2023-10-19 07:18:22 -05:00
|
|
|
if (!getImageSizeWorker) {
|
|
|
|
getImageSizeWorker = new Piscina({
|
|
|
|
filename: new URL(join('workers', 'get-image-size.js'), import.meta.url).href,
|
|
|
|
concurrentTasksPerWorker: WORKER_THREADS.GET_IMAGE_SIZE.CONCURRENCY,
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: WORKER_THREADS.GET_IMAGE_SIZE.MAX_THREADS,
|
|
|
|
minThreads: 1
|
2023-10-19 07:18:22 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
getImageSizeWorker.on('error', err => logger.error('Error in get image size worker', { err }))
|
2023-10-19 07:18:22 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return getImageSizeWorker.run(options)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2022-09-08 05:26:46 -05:00
|
|
|
let parallelHTTPBroadcastWorker: Piscina
|
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
export function parallelHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadcast>[0]): Promise<ReturnType<typeof httpBroadcast>> {
|
2022-09-08 05:26:46 -05:00
|
|
|
if (!parallelHTTPBroadcastWorker) {
|
|
|
|
parallelHTTPBroadcastWorker = new Piscina({
|
2023-07-31 07:34:36 -05:00
|
|
|
filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
|
2022-11-16 07:40:10 -06:00
|
|
|
// Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
|
|
|
|
concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'],
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: 1,
|
|
|
|
minThreads: 1
|
2022-09-08 05:26:46 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
parallelHTTPBroadcastWorker.on('error', err => logger.error('Error in parallel HTTP broadcast worker', { err }))
|
2022-09-08 05:26:46 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return parallelHTTPBroadcastWorker.run(options)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
let sequentialHTTPBroadcastWorker: Piscina
|
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
export function sequentialHTTPBroadcastFromWorker (
|
|
|
|
options: Parameters<typeof httpBroadcast>[0]
|
|
|
|
): Promise<ReturnType<typeof httpBroadcast>> {
|
2022-09-08 05:26:46 -05:00
|
|
|
if (!sequentialHTTPBroadcastWorker) {
|
|
|
|
sequentialHTTPBroadcastWorker = new Piscina({
|
2023-07-31 07:34:36 -05:00
|
|
|
filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
|
2022-11-16 07:40:10 -06:00
|
|
|
// Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
|
|
|
|
concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'],
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: 1,
|
|
|
|
minThreads: 1
|
2022-09-08 05:26:46 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
sequentialHTTPBroadcastWorker.on('error', err => logger.error('Error in sequential HTTP broadcast image worker', { err }))
|
2022-09-08 05:26:46 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return sequentialHTTPBroadcastWorker.run(options)
|
|
|
|
}
|
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2023-10-25 02:43:35 -05:00
|
|
|
let httpUnicastWorker: Piscina
|
|
|
|
|
|
|
|
export function httpUnicastFromWorker (
|
|
|
|
options: Parameters<typeof httpUnicast>[0]
|
|
|
|
): Promise<ReturnType<typeof httpUnicast>> {
|
|
|
|
if (!httpUnicastWorker) {
|
|
|
|
httpUnicastWorker = new Piscina({
|
|
|
|
filename: new URL(join('workers', 'http-unicast.js'), import.meta.url).href,
|
|
|
|
// Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
|
|
|
|
concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-unicast'],
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: 1,
|
|
|
|
minThreads: 1
|
2023-10-25 02:43:35 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
httpUnicastWorker.on('error', err => logger.error('Error in HTTP unicast worker', { err }))
|
2023-10-25 02:43:35 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return httpUnicastWorker.run(options)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2023-10-24 03:45:17 -05:00
|
|
|
let signJsonLDObjectWorker: Piscina
|
|
|
|
|
|
|
|
export function signJsonLDObjectFromWorker <T> (
|
|
|
|
options: Parameters<typeof signJsonLDObject<T>>[0]
|
|
|
|
): ReturnType<typeof signJsonLDObject<T>> {
|
|
|
|
if (!signJsonLDObjectWorker) {
|
|
|
|
signJsonLDObjectWorker = new Piscina({
|
|
|
|
filename: new URL(join('workers', 'sign-json-ld-object.js'), import.meta.url).href,
|
|
|
|
concurrentTasksPerWorker: WORKER_THREADS.SIGN_JSON_LD_OBJECT.CONCURRENCY,
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: WORKER_THREADS.SIGN_JSON_LD_OBJECT.MAX_THREADS,
|
|
|
|
minThreads: 1
|
2023-10-24 03:45:17 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
signJsonLDObjectWorker.on('error', err => logger.error('Error in sign JSONLD object worker', { err }))
|
2023-10-24 03:45:17 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return signJsonLDObjectWorker.run(options)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
let buildDigestWorker: Piscina
|
|
|
|
|
|
|
|
export function buildDigestFromWorker (
|
|
|
|
options: Parameters<typeof buildDigest>[0]
|
|
|
|
): Promise<ReturnType<typeof buildDigest>> {
|
|
|
|
if (!buildDigestWorker) {
|
|
|
|
buildDigestWorker = new Piscina({
|
|
|
|
filename: new URL(join('workers', 'build-digest.js'), import.meta.url).href,
|
|
|
|
// Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
|
|
|
|
concurrentTasksPerWorker: WORKER_THREADS.BUILD_DIGEST.CONCURRENCY,
|
2023-11-28 06:32:51 -06:00
|
|
|
maxThreads: WORKER_THREADS.BUILD_DIGEST.MAX_THREADS,
|
|
|
|
minThreads: 1
|
2023-10-24 03:45:17 -05:00
|
|
|
})
|
2023-11-21 10:29:20 -06:00
|
|
|
|
|
|
|
buildDigestWorker.on('error', err => logger.error('Error in build digest worker', { err }))
|
2023-10-24 03:45:17 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return buildDigestWorker.run(options)
|
2022-06-24 09:31:32 -05:00
|
|
|
}
|
2023-10-26 09:34:54 -05:00
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
2023-10-27 07:40:53 -05:00
|
|
|
export function getWorkersStats () {
|
2023-10-26 09:34:54 -05:00
|
|
|
return [
|
|
|
|
{
|
|
|
|
label: 'downloadImage',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: downloadImageWorker?.queueSize || 0,
|
|
|
|
completed: downloadImageWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'processImageWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: processImageWorker?.queueSize || 0,
|
|
|
|
completed: processImageWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'getImageSizeWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: getImageSizeWorker?.queueSize || 0,
|
|
|
|
completed: getImageSizeWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'parallelHTTPBroadcastWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: parallelHTTPBroadcastWorker?.queueSize || 0,
|
|
|
|
completed: parallelHTTPBroadcastWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'sequentialHTTPBroadcastWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: sequentialHTTPBroadcastWorker?.queueSize || 0,
|
|
|
|
completed: sequentialHTTPBroadcastWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'httpUnicastWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: httpUnicastWorker?.queueSize || 0,
|
|
|
|
completed: httpUnicastWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'signJsonLDObjectWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: signJsonLDObjectWorker?.queueSize || 0,
|
|
|
|
completed: signJsonLDObjectWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
},
|
|
|
|
{
|
|
|
|
label: 'buildDigestWorker',
|
2023-10-27 07:40:53 -05:00
|
|
|
queueSize: buildDigestWorker?.queueSize || 0,
|
|
|
|
completed: buildDigestWorker?.completed || 0
|
2023-10-26 09:34:54 -05:00
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|