From 83f8ea5c14d70a789e8c9982812120fdd0c85285 Mon Sep 17 00:00:00 2001 From: Henri BAUDESSON Date: Tue, 19 Dec 2023 16:02:00 +0100 Subject: [PATCH] Runner download videoFileUrl follow redirect The runner downloads the video file from the url set in the paylaod of a transcoding job. This url is pointing to our API and the runner will make POST request to it with an jobToken and a runnerToken. Doing this ensure we can verify the tokens and return the video file. But returning the video file also means that we are using server resources to serve the file. If the runner is able to follow the redirect, we can do our usual verification and return a redirect response to the url of the video file, the runner will download it using his own resources. --- apps/peertube-runner/package.json | 5 +- apps/peertube-runner/src/shared/http.ts | 8 +- apps/peertube-runner/yarn.lock | 162 ++++-------------------- 3 files changed, 32 insertions(+), 143 deletions(-) diff --git a/apps/peertube-runner/package.json b/apps/peertube-runner/package.json index 8389a5a01..b043e89cc 100644 --- a/apps/peertube-runner/package.json +++ b/apps/peertube-runner/package.json @@ -8,10 +8,13 @@ "node": ">=16.x" }, "license": "AGPL-3.0", - "dependencies": {}, + "dependencies": { + "follow-redirects": "1.15.3" + }, "devDependencies": { "@commander-js/extra-typings": "^10.0.3", "@iarna/toml": "^2.2.5", + "@types/follow-redirects": "1.14.4", "env-paths": "^3.0.0", "net-ipc": "^2.0.1", "pino": "^8.11.0", diff --git a/apps/peertube-runner/src/shared/http.ts b/apps/peertube-runner/src/shared/http.ts index 42886279c..4d86c2ef7 100644 --- a/apps/peertube-runner/src/shared/http.ts +++ b/apps/peertube-runner/src/shared/http.ts @@ -1,7 +1,7 @@ import { createWriteStream } from 'fs' import { remove } from 'fs-extra/esm' -import { request as requestHTTP } from 'http' -import { request as requestHTTPS, RequestOptions } from 'https' +import { RequestOptions } from 'https' +import { http, https } from 'follow-redirects' import { logger } from './logger.js' export function downloadFile (options: { @@ -61,7 +61,7 @@ export function downloadFile (options: { // --------------------------------------------------------------------------- function getRequest (url: string) { - if (url.startsWith('https://')) return requestHTTPS + if (url.startsWith('https://')) return https.request - return requestHTTP + return http.request } diff --git a/apps/peertube-runner/yarn.lock b/apps/peertube-runner/yarn.lock index adb5aa118..0955b263b 100644 --- a/apps/peertube-runner/yarn.lock +++ b/apps/peertube-runner/yarn.lock @@ -7,116 +7,6 @@ resolved "https://registry.yarnpkg.com/@commander-js/extra-typings/-/extra-typings-10.0.3.tgz#8b6c64897231ed9c00461db82018b5131b653aae" integrity sha512-OIw28QV/GlP8k0B5CJTRsl8IyNvd0R8C8rfo54Yz9P388vCNDgdNrFlKxZTGqps+5j6lSw3Ss9JTQwcur1w1oA== -"@esbuild/android-arm64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.15.tgz#893ad71f3920ccb919e1757c387756a9bca2ef42" - integrity sha512-0kOB6Y7Br3KDVgHeg8PRcvfLkq+AccreK///B4Z6fNZGr/tNHX0z2VywCc7PTeWp+bPvjA5WMvNXltHw5QjAIA== - -"@esbuild/android-arm@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.17.15.tgz#143e0d4e4c08c786ea410b9a7739779a9a1315d8" - integrity sha512-sRSOVlLawAktpMvDyJIkdLI/c/kdRTOqo8t6ImVxg8yT7LQDUYV5Rp2FKeEosLr6ZCja9UjYAzyRSxGteSJPYg== - -"@esbuild/android-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.17.15.tgz#d2d12a7676b2589864281b2274355200916540bc" - integrity sha512-MzDqnNajQZ63YkaUWVl9uuhcWyEyh69HGpMIrf+acR4otMkfLJ4sUCxqwbCyPGicE9dVlrysI3lMcDBjGiBBcQ== - -"@esbuild/darwin-arm64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.17.15.tgz#2e88e79f1d327a2a7d9d06397e5232eb0a473d61" - integrity sha512-7siLjBc88Z4+6qkMDxPT2juf2e8SJxmsbNVKFY2ifWCDT72v5YJz9arlvBw5oB4W/e61H1+HDB/jnu8nNg0rLA== - -"@esbuild/darwin-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.17.15.tgz#9384e64c0be91388c57be6d3a5eaf1c32a99c91d" - integrity sha512-NbImBas2rXwYI52BOKTW342Tm3LTeVlaOQ4QPZ7XuWNKiO226DisFk/RyPk3T0CKZkKMuU69yOvlapJEmax7cg== - -"@esbuild/freebsd-arm64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.15.tgz#2ad5a35bc52ebd9ca6b845dbc59ba39647a93c1a" - integrity sha512-Xk9xMDjBVG6CfgoqlVczHAdJnCs0/oeFOspFap5NkYAmRCT2qTn1vJWA2f419iMtsHSLm+O8B6SLV/HlY5cYKg== - -"@esbuild/freebsd-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.17.15.tgz#b513a48446f96c75fda5bef470e64d342d4379cd" - integrity sha512-3TWAnnEOdclvb2pnfsTWtdwthPfOz7qAfcwDLcfZyGJwm1SRZIMOeB5FODVhnM93mFSPsHB9b/PmxNNbSnd0RQ== - -"@esbuild/linux-arm64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.17.15.tgz#9697b168175bfd41fa9cc4a72dd0d48f24715f31" - integrity sha512-T0MVnYw9KT6b83/SqyznTs/3Jg2ODWrZfNccg11XjDehIved2oQfrX/wVuev9N936BpMRaTR9I1J0tdGgUgpJA== - -"@esbuild/linux-arm@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.17.15.tgz#5b22062c54f48cd92fab9ffd993732a52db70cd3" - integrity sha512-MLTgiXWEMAMr8nmS9Gigx43zPRmEfeBfGCwxFQEMgJ5MC53QKajaclW6XDPjwJvhbebv+RzK05TQjvH3/aM4Xw== - -"@esbuild/linux-ia32@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.17.15.tgz#eb28a13f9b60b5189fcc9e98e1024f6b657ba54c" - integrity sha512-wp02sHs015T23zsQtU4Cj57WiteiuASHlD7rXjKUyAGYzlOKDAjqK6bk5dMi2QEl/KVOcsjwL36kD+WW7vJt8Q== - -"@esbuild/linux-loong64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.17.15.tgz#32454bdfe144cf74b77895a8ad21a15cb81cfbe5" - integrity sha512-k7FsUJjGGSxwnBmMh8d7IbObWu+sF/qbwc+xKZkBe/lTAF16RqxRCnNHA7QTd3oS2AfGBAnHlXL67shV5bBThQ== - -"@esbuild/linux-mips64el@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.17.15.tgz#af12bde0d775a318fad90eb13a0455229a63987c" - integrity sha512-ZLWk6czDdog+Q9kE/Jfbilu24vEe/iW/Sj2d8EVsmiixQ1rM2RKH2n36qfxK4e8tVcaXkvuV3mU5zTZviE+NVQ== - -"@esbuild/linux-ppc64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.17.15.tgz#34c5ed145b2dfc493d3e652abac8bd3baa3865a5" - integrity sha512-mY6dPkIRAiFHRsGfOYZC8Q9rmr8vOBZBme0/j15zFUKM99d4ILY4WpOC7i/LqoY+RE7KaMaSfvY8CqjJtuO4xg== - -"@esbuild/linux-riscv64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.17.15.tgz#87bd515e837f2eb004b45f9e6a94dc5b93f22b92" - integrity sha512-EcyUtxffdDtWjjwIH8sKzpDRLcVtqANooMNASO59y+xmqqRYBBM7xVLQhqF7nksIbm2yHABptoioS9RAbVMWVA== - -"@esbuild/linux-s390x@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.17.15.tgz#20bf7947197f199ddac2ec412029a414ceae3aa3" - integrity sha512-BuS6Jx/ezxFuHxgsfvz7T4g4YlVrmCmg7UAwboeyNNg0OzNzKsIZXpr3Sb/ZREDXWgt48RO4UQRDBxJN3B9Rbg== - -"@esbuild/linux-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.17.15.tgz#31b93f9c94c195e852c20cd3d1914a68aa619124" - integrity sha512-JsdS0EgEViwuKsw5tiJQo9UdQdUJYuB+Mf6HxtJSPN35vez1hlrNb1KajvKWF5Sa35j17+rW1ECEO9iNrIXbNg== - -"@esbuild/netbsd-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.17.15.tgz#8da299b3ac6875836ca8cdc1925826498069ac65" - integrity sha512-R6fKjtUysYGym6uXf6qyNephVUQAGtf3n2RCsOST/neIwPqRWcnc3ogcielOd6pT+J0RDR1RGcy0ZY7d3uHVLA== - -"@esbuild/openbsd-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.17.15.tgz#04a1ec3d4e919714dba68dcf09eeb1228ad0d20c" - integrity sha512-mVD4PGc26b8PI60QaPUltYKeSX0wxuy0AltC+WCTFwvKCq2+OgLP4+fFd+hZXzO2xW1HPKcytZBdjqL6FQFa7w== - -"@esbuild/sunos-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.17.15.tgz#6694ebe4e16e5cd7dab6505ff7c28f9c1c695ce5" - integrity sha512-U6tYPovOkw3459t2CBwGcFYfFRjivcJJc1WC8Q3funIwX8x4fP+R6xL/QuTPNGOblbq/EUDxj9GU+dWKX0oWlQ== - -"@esbuild/win32-arm64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.17.15.tgz#1f95b2564193c8d1fee8f8129a0609728171d500" - integrity sha512-W+Z5F++wgKAleDABemiyXVnzXgvRFs+GVKThSI+mGgleLWluv0D7Diz4oQpgdpNzh4i2nNDzQtWbjJiqutRp6Q== - -"@esbuild/win32-ia32@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.17.15.tgz#c362b88b3df21916ed7bcf75c6d09c6bf3ae354a" - integrity sha512-Muz/+uGgheShKGqSVS1KsHtCyEzcdOn/W/Xbh6H91Etm+wiIfwZaBn1W58MeGtfI8WA961YMHFYTthBdQs4t+w== - -"@esbuild/win32-x64@0.17.15": - version "0.17.15" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.17.15.tgz#c2e737f3a201ebff8e2ac2b8e9f246b397ad19b8" - integrity sha512-DjDa9ywLUUmjhV2Y9wUTIF+1XsmuFGvZoCmOWkli1XcNAh5t25cc7fgsCx4Zi/Uurep3TTLyDiKATgGEg61pkA== - "@iarna/toml@^2.2.5": version "2.2.5" resolved "https://registry.yarnpkg.com/@iarna/toml/-/toml-2.2.5.tgz#b32366c89b43c6f8cefbdefac778b9c828e3ba8c" @@ -152,6 +42,20 @@ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz#0f164b726869f71da3c594171df5ebc1c4b0a407" integrity sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ== +"@types/follow-redirects@1.14.4": + version "1.14.4" + resolved "https://registry.yarnpkg.com/@types/follow-redirects/-/follow-redirects-1.14.4.tgz#ca054d72ef574c77949fc5fff278b430fcd508ec" + integrity sha512-GWXfsD0Jc1RWiFmMuMFCpXMzi9L7oPDVwxUnZdg89kDNnqsRfUKXEtUYtA98A6lig1WXH/CYY/fvPW9HuN5fTA== + dependencies: + "@types/node" "*" + +"@types/node@*": + version "20.10.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2" + integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw== + dependencies: + undici-types "~5.26.4" + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -211,34 +115,6 @@ env-paths@^3.0.0: resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-3.0.0.tgz#2f1e89c2f6dbd3408e1b1711dd82d62e317f58da" integrity sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A== -esbuild@^0.17.15: - version "0.17.15" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.17.15.tgz#209ebc87cb671ffb79574db93494b10ffaf43cbc" - integrity sha512-LBUV2VsUIc/iD9ME75qhT4aJj0r75abCVS0jakhFzOtR7TQsqQA5w0tZ+KTKnwl3kXE0MhskNdHDh/I5aCR1Zw== - optionalDependencies: - "@esbuild/android-arm" "0.17.15" - "@esbuild/android-arm64" "0.17.15" - "@esbuild/android-x64" "0.17.15" - "@esbuild/darwin-arm64" "0.17.15" - "@esbuild/darwin-x64" "0.17.15" - "@esbuild/freebsd-arm64" "0.17.15" - "@esbuild/freebsd-x64" "0.17.15" - "@esbuild/linux-arm" "0.17.15" - "@esbuild/linux-arm64" "0.17.15" - "@esbuild/linux-ia32" "0.17.15" - "@esbuild/linux-loong64" "0.17.15" - "@esbuild/linux-mips64el" "0.17.15" - "@esbuild/linux-ppc64" "0.17.15" - "@esbuild/linux-riscv64" "0.17.15" - "@esbuild/linux-s390x" "0.17.15" - "@esbuild/linux-x64" "0.17.15" - "@esbuild/netbsd-x64" "0.17.15" - "@esbuild/openbsd-x64" "0.17.15" - "@esbuild/sunos-x64" "0.17.15" - "@esbuild/win32-arm64" "0.17.15" - "@esbuild/win32-ia32" "0.17.15" - "@esbuild/win32-x64" "0.17.15" - event-target-shim@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" @@ -269,6 +145,11 @@ fast-zlib@^2.0.1: resolved "https://registry.yarnpkg.com/fast-zlib/-/fast-zlib-2.0.1.tgz#be624f592fc80ad8019ee2025d16a367a4e9b024" integrity sha512-DCoYgNagM2Bt1VIpXpdGnRx4LzqJeYG0oh6Nf/7cWo6elTXkFGMw9CrRCYYUIapYNrozYMoyDRflx9mgT3Awyw== +follow-redirects@1.15.3: + version "1.15.3" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" + integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -517,6 +398,11 @@ thread-stream@^2.0.0: dependencies: real-require "^0.2.0" +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + util-deprecate@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"