interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Upgrade redis dep

This commit is contained in:
Chocobozzz 2021-12-24 14:49:03 +01:00
parent 2d28b0c21d
commit e5d91a9b9c
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
5 changed files with 120 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@ -131,7 +131,7 @@
"prompt": "^1.0.0",
"proxy-addr": "^2.0.7",
"pug": "^3.0.0",
"redis": "^3.0.2",
"redis": "^4.0.1",
"reflect-metadata": "^0.1.12",
"sanitize-html": "2.x",
"sequelize": "6.9.0",
@ -182,7 +182,6 @@
"@types/nodemailer": "^6.2.0",
"@types/oauth2-server": "^3.0.8",
"@types/pem": "^1.9.3",
"@types/redis": "^2.8.5",
"@types/request": "^2.0.3",
"@types/supertest": "^2.0.3",
"@types/validator": "^13.0.0",

8
server/lib/plugins/plugin-manager.ts
View File

@ -5,7 +5,13 @@ import { basename, join } from 'path'
import { decachePlugin } from '@server/helpers/decache'
import { MOAuthTokenUser, MUser } from '@server/types/models'
import { getCompleteLocale } from '@shared/core-utils'
import { ClientScriptJSON, PluginPackageJSON, PluginTranslation, PluginTranslationPathsJSON, RegisterServerHookOptions } from '@shared/models'
import {
ClientScriptJSON,
PluginPackageJSON,
PluginTranslation,
PluginTranslationPathsJSON,
RegisterServerHookOptions
} from '@shared/models'
import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks'
import { PluginType } from '../../../shared/models/plugins/plugin.type'
import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model'

172
server/lib/redis.ts
View File

@ -1,31 +1,29 @@
import express from 'express'
import { createClient, RedisClient } from 'redis'
import { createClient } from 'redis'
import { exists } from '@server/helpers/custom-validators/misc'
import { logger } from '../helpers/logger'
import { generateRandomString } from '../helpers/utils'
import { CONFIG } from '../initializers/config'
import {
CONTACT_FORM_LIFETIME,
USER_EMAIL_VERIFY_LIFETIME,
USER_PASSWORD_RESET_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
VIEW_LIFETIME,
WEBSERVER,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
TRACKER_RATE_LIMITS,
RESUMABLE_UPLOAD_SESSION_LIFETIME
USER_EMAIL_VERIFY_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
USER_PASSWORD_RESET_LIFETIME,
VIEW_LIFETIME,
WEBSERVER
} from '../initializers/constants'
import { CONFIG } from '../initializers/config'
import { exists } from '@server/helpers/custom-validators/misc'
type CachedRoute = {
body: string
contentType?: string
statusCode?: string
}
// Only used for typings
const redisClientWrapperForType = () => createClient<{}>()
class Redis {
private static instance: Redis
private initialized = false
private client: RedisClient
private connected = false
private client: ReturnType<typeof redisClientWrapperForType>
private prefix: string
private constructor () {
@ -38,21 +36,24 @@ class Redis {
this.client = createClient(Redis.getRedisClientOptions())
this.client.connect()
.then(() => { this.connected = true })
.catch(err => {
logger.error('Cannot connect to redis', { err })
process.exit(-1)
})
this.client.on('error', err => {
logger.error('Error in Redis client.', { err })
process.exit(-1)
})
if (CONFIG.REDIS.AUTH) {
this.client.auth(CONFIG.REDIS.AUTH)
}
this.prefix = 'redis-' + WEBSERVER.HOST + '-'
}
static getRedisClientOptions () {
return Object.assign({},
(CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {},
CONFIG.REDIS.AUTH ? { password: CONFIG.REDIS.AUTH } : {},
(CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {},
(CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT)
? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
@ -68,6 +69,10 @@ class Redis {
return this.prefix
}
isConnected () {
return this.connected
}
/* ************ Forgot password ************ */
async setResetPasswordVerificationString (userId: number) {
@ -146,25 +151,6 @@ class Redis {
return this.exists(this.generateTrackerBlockIPKey(ip))
}
/* ************ API cache ************ */
async getCachedRoute (req: express.Request) {
const cached = await this.getObject(this.generateCachedRouteKey(req))
return cached as CachedRoute
}
setCachedRoute (req: express.Request, body: any, lifetime: number, contentType?: string, statusCode?: number) {
const cached: CachedRoute = Object.assign(
{},
{ body: body.toString() },
(contentType) ? { contentType } : null,
(statusCode) ? { statusCode: statusCode.toString() } : null
)
return this.setObject(this.generateCachedRouteKey(req), cached, lifetime)
}
/* ************ Video views stats ************ */
addVideoViewStats (videoId: number) {
@ -277,10 +263,6 @@ class Redis {
/* ************ Keys generation ************ */
generateCachedRouteKey (req: express.Request) {
return req.method + '-' + req.originalUrl
}
private generateLocalVideoViewsKeys (videoId?: Number) {
return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
}
@ -320,125 +302,45 @@ class Redis {
/* ************ Redis helpers ************ */
private getValue (key: string) {
return new Promise<string>((res, rej) => {
this.client.get(this.prefix + key, (err, value) => {
if (err) return rej(err)
return res(value)
})
})
return this.client.get(this.prefix + key)
}
private getSet (key: string) {
return new Promise<string[]>((res, rej) => {
this.client.smembers(this.prefix + key, (err, value) => {
if (err) return rej(err)
return res(value)
})
})
return this.client.sMembers(this.prefix + key)
}
private addToSet (key: string, value: string) {
return new Promise<void>((res, rej) => {
this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res())
})
return this.client.sAdd(this.prefix + key, value)
}
private deleteFromSet (key: string, value: string) {
return new Promise<void>((res, rej) => {
this.client.srem(this.prefix + key, value, err => err ? rej(err) : res())
})
return this.client.sRem(this.prefix + key, value)
}
private deleteKey (key: string) {
return new Promise<void>((res, rej) => {
this.client.del(this.prefix + key, err => err ? rej(err) : res())
})
return this.client.del(this.prefix + key)
}
private deleteFieldInHash (key: string, field: string) {
return new Promise<void>((res, rej) => {
this.client.hdel(this.prefix + key, field, err => err ? rej(err) : res())
})
}
private async setValue (key: string, value: string, expirationMilliseconds: number) {
const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds })
private setValue (key: string, value: string, expirationMilliseconds: number) {
return new Promise<void>((res, rej) => {
this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => {
if (err) return rej(err)
if (ok !== 'OK') return rej(new Error('Redis set result is not OK.'))
return res()
})
})
if (result !== 'OK') throw new Error('Redis set result is not OK.')
}
private removeValue (key: string) {
return new Promise<void>((res, rej) => {
this.client.del(this.prefix + key, err => {
if (err) return rej(err)
return res()
})
})
}
private setObject (key: string, obj: { [id: string]: string }, expirationMilliseconds: number) {
return new Promise<void>((res, rej) => {
this.client.hmset(this.prefix + key, obj, (err, ok) => {
if (err) return rej(err)
if (!ok) return rej(new Error('Redis mset result is not OK.'))
this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => {
if (err) return rej(err)
if (!ok) return rej(new Error('Redis expiration result is not OK.'))
return res()
})
})
})
return this.client.del(this.prefix + key)
}
private getObject (key: string) {
return new Promise<{ [id: string]: string }>((res, rej) => {
this.client.hgetall(this.prefix + key, (err, value) => {
if (err) return rej(err)
return res(value)
})
})
}
private setValueInHash (key: string, field: string, value: string) {
return new Promise<void>((res, rej) => {
this.client.hset(this.prefix + key, field, value, (err) => {
if (err) return rej(err)
return res()
})
})
return this.client.hGetAll(this.prefix + key)
}
private increment (key: string) {
return new Promise<number>((res, rej) => {
this.client.incr(this.prefix + key, (err, value) => {
if (err) return rej(err)
return res(value)
})
})
return this.client.incr(this.prefix + key)
}
private exists (key: string) {
return new Promise<boolean>((res, rej) => {
this.client.exists(this.prefix + key, (err, existsNumber) => {
if (err) return rej(err)
return res(existsNumber === 1)
})
})
return this.client.exists(this.prefix + key)
}
static get Instance () {

50
server/middlewares/cache/shared/api-cache.ts vendored
View File

@ -7,6 +7,7 @@ import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils'
import { logger } from '@server/helpers/logger'
import { Redis } from '@server/lib/redis'
import { HttpStatusCode } from '@shared/models'
import { asyncMiddleware } from '@server/middlewares'
export interface APICacheOptions {
headerBlacklist?: string[]
@ -40,24 +41,25 @@ export class ApiCache {
buildMiddleware (strDuration: string) {
const duration = parseDurationToMs(strDuration)
return (req: express.Request, res: express.Response, next: express.NextFunction) => {
const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
const redis = Redis.Instance.getClient()
return asyncMiddleware(
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
const redis = Redis.Instance.getClient()
if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration)
if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration)
try {
redis.hgetall(key, (err, obj) => {
if (!err && obj && obj.response) {
try {
const obj = await redis.hGetAll(key)
if (obj?.response) {
return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration)
}
return this.makeResponseCacheable(res, next, key, duration)
})
} catch (err) {
return this.makeResponseCacheable(res, next, key, duration)
} catch (err) {
return this.makeResponseCacheable(res, next, key, duration)
}
}
}
)
}
private shouldCacheResponse (response: express.Response) {
@ -93,21 +95,22 @@ export class ApiCache {
} as CacheObject
}
private cacheResponse (key: string, value: object, duration: number) {
private async cacheResponse (key: string, value: object, duration: number) {
const redis = Redis.Instance.getClient()
if (redis.connected) {
try {
redis.hset(key, 'response', JSON.stringify(value))
redis.hset(key, 'duration', duration + '')
if (Redis.Instance.isConnected()) {
await Promise.all([
redis.hSet(key, 'response', JSON.stringify(value)),
redis.hSet(key, 'duration', duration + ''),
redis.expire(key, duration / 1000)
} catch (err) {
logger.error('Cannot set cache in redis.', { err })
}
])
}
// add automatic cache clearing from duration, includes max limit on setTimeout
this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647))
this.timers[key] = setTimeout(() => {
this.clear(key)
.catch(err => logger.error('Cannot clear Redis key %s.', key, { err }))
}, Math.min(duration, 2147483647))
}
private accumulateContent (res: express.Response, content: any) {
@ -184,6 +187,7 @@ export class ApiCache {
encoding
)
self.cacheResponse(key, cacheObject, duration)
.catch(err => logger.error('Cannot cache response', { err }))
}
}
@ -235,7 +239,7 @@ export class ApiCache {
return response.end(data, cacheObject.encoding)
}
private clear (target: string) {
private async clear (target: string) {
const redis = Redis.Instance.getClient()
if (target) {
@ -243,7 +247,7 @@ export class ApiCache {
delete this.timers[target]
try {
redis.del(target)
await redis.del(target)
} catch (err) {
logger.error('Cannot delete %s in redis cache.', target, { err })
}
@ -255,7 +259,7 @@ export class ApiCache {
delete this.timers[key]
try {
redis.del(key)
await redis.del(key)
} catch (err) {
logger.error('Cannot delete %s in redis cache.', key, { err })
}

66
yarn.lock
View File

@ -1365,6 +1365,31 @@
semver "^7.3.5"
tar "^6.1.11"
"@node-redis/client@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.1.tgz#ddca6021097ce1026fedc193cac8c36b05c6cad8"
integrity sha512-o0I4LdzJXP6QYxRnBPrQ7cIG5tF3SNM/PBnjC3mV6QkzIiGRElzWqSr9a9JCJdcyB1SIA80bhgGhpdTpCQ1Sdw==
dependencies:
cluster-key-slot "1.1.0"
generic-pool "3.8.2"
redis-parser "3.0.0"
yallist "4.0.0"
"@node-redis/json@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.1.tgz#8cd987c1855392adf21bc4f06163a7eda97a40a3"
integrity sha512-2EB96ZN0yUr4mgA9Odme48jX8eF5Ji0jrsRn4rLfEhME7L3rHLdKeUfxJKxbPOxadP6k8+6ViElxPZrKuV2nvQ==
"@node-redis/search@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.1.tgz#8d0936049f4858b9aefab40524ce8e5a52e5d08e"
integrity sha512-iA2Gw6gr0X6IfNSjTyme9W1tDlLkwQ1bPApo4s8aVwZ2Ju8Z4COVik0vT6BJPRin79f5xPZgnaec3DIoC2UpHA==
"@node-redis/time-series@^1.0.0":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.0.tgz#3db4caa63d7c158f0b73ab6cd46bd6c9c187dfaf"
integrity sha512-QcaCIL/DlYJXedSfmjF+IRxKJbBUXBrjA5Gv0IiPlXXFFOkRnbPGKq6hmwBAAWyk1U03wyBHDFKVS3/9GnZV8g==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@ -1805,7 +1830,7 @@
resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
"@types/redis@^2.8.0", "@types/redis@^2.8.5":
"@types/redis@^2.8.0":
version "2.8.32"
resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
@ -2995,7 +3020,7 @@ clone@^2.0.0:
resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
cluster-key-slot@^1.1.0:
cluster-key-slot@1.1.0, cluster-key-slot@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
@ -3475,7 +3500,7 @@ delegates@^1.0.0:
resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
denque@^1.1.0, denque@^1.5.0:
denque@^1.1.0:
version "1.5.1"
resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
@ -4479,6 +4504,11 @@ gauge@^3.0.0:
strip-ansi "^6.0.1"
wide-align "^1.1.2"
generic-pool@3.8.2:
version "3.8.2"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
get-browser-rtc@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz#d1494e299b00f33fc8e9d6d3343ba4ba99711a2c"
@ -7352,7 +7382,7 @@ record-cache@^1.0.2:
resolved "https://registry.yarnpkg.com/record-cache/-/record-cache-1.1.1.tgz#ba3088a489f50491a4af7b14d410822c394fb811"
integrity sha512-L5hZlgWc7CmGbztnemQoKE1bLu9rtI2skOB0ttE4C5+TVszLE8Rd0YLTROSgvXKLAqPumS/soyN5tJW5wJLmJQ==
redis-commands@1.7.0, redis-commands@^1.7.0:
redis-commands@1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
@ -7362,22 +7392,22 @@ redis-errors@^1.0.0, redis-errors@^1.2.0:
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
redis-parser@^3.0.0:
redis-parser@3.0.0, redis-parser@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
dependencies:
redis-errors "^1.0.0"
redis@^3.0.2:
version "3.1.2"
resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c"
integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==
redis@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.1.tgz#c020e2ac7f83f0c1d42ced50b8a7af28164bd6ee"
integrity sha512-qfcq1oz2ci7pNdCfTLLEuKhS8jZ17dFiT1exogOr+jd3EVP/h9qpy7K+VajB4BXA0k8q68KFqR6HrliKV6jt1Q==
dependencies:
denque "^1.5.0"
redis-commands "^1.7.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
"@node-redis/client" "^1.0.1"
"@node-redis/json" "^1.0.1"
"@node-redis/search" "^1.0.1"
"@node-redis/time-series" "^1.0.0"
reflect-metadata@^0.1.12:
version "0.1.13"
@ -9050,16 +9080,16 @@ y18n@^5.0.5:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
yallist@4.0.0, yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yaml@^1.10.0:
version "1.10.2"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"