interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Check live duration and size

This commit is contained in:
Chocobozzz 2020-09-25 16:19:35 +02:00 committed by Chocobozzz
parent a5cf76afa3
commit fb7194043d
27 changed files with 436 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
View File

@ -699,7 +699,7 @@
</ng-template>
</ng-container>
<ng-container ngbNavItem="live">
<div ngbNavItem="live">
<a ngbNavLink i18n>Live streaming</a>
<ng-template ngbNavContent>
@ -722,54 +722,78 @@
<ng-container i18n>Allow live streaming</ng-container>
</ng-template>
<ng-template ptTemplate="help">
<ng-container i18n>Enabling live streaming requires trust in your users and extra moderation work</ng-container>
</ng-template>
<ng-container ngProjectAs="description" i18n>
⚠️ Enabling live streaming requires trust in your users and extra moderation work
</ng-container>
<ng-container ngProjectAs="extra" formGroupName="transcoding">
<ng-container ngProjectAs="extra">
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() }">
<my-peertube-checkbox
inputName="liveTranscodingEnabled" formControlName="enabled"
i18n-labelText labelText="Enable live transcoding"
inputName="liveAllowReplay" formControlName="allowReplay"
i18n-labelText labelText="Allow your users to automatically publish a replay of their live"
>
<ng-container ngProjectAs="description">
Requires a lot of CPU!
<ng-container ngProjectAs="description" i18n>
If the user quota is reached, PeerTube will automatically terminate the live streaming
</ng-container>
</my-peertube-checkbox>
</div>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() || !isLiveTranscodingEnabled() }">
<label i18n for="liveTranscodingThreads">Live transcoding threads</label>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() }">
<label i18n for="liveMaxDuration">Max live duration</label>
<div class="peertube-select-container">
<select id="liveTranscodingThreads" formControlName="threads" class="form-control">
<option *ngFor="let transcodingThreadOption of transcodingThreadOptions" [value]="transcodingThreadOption.value">
{{ transcodingThreadOption.label }}
<select id="liveMaxDuration" formControlName="maxDuration" class="form-control">
<option *ngFor="let liveMaxDurationOption of liveMaxDurationOptions" [value]="liveMaxDurationOption.value">
{{ liveMaxDurationOption.label }}
</option>
</select>
</div>
<div *ngIf="formErrors.live.transcoding.threads" class="form-error">{{ formErrors.live.transcoding.threads }}</div>
</div>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() || !isLiveTranscodingEnabled() }">
<ng-container formGroupName="transcoding">
<label i18n for="liveTranscodingThreads">Live resolutions to generate</label>
<div class="ml-2 mt-2 d-flex flex-column">
<ng-container formGroupName="resolutions">
<div class="form-group" *ngFor="let resolution of liveResolutions">
<my-peertube-checkbox
[inputName]="getResolutionKey(resolution.id)" [formControlName]="resolution.id"
labelText="{{resolution.label}}"
>
<ng-template *ngIf="resolution.description" ptTemplate="help">
<div [innerHTML]="resolution.description"></div>
</ng-template>
</my-peertube-checkbox>
</div>
</ng-container>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() }">
<my-peertube-checkbox
inputName="liveTranscodingEnabled" formControlName="enabled"
i18n-labelText labelText="Enable live transcoding"
>
<ng-container ngProjectAs="description" i18n>
Requires a lot of CPU!
</ng-container>
</my-peertube-checkbox>
</div>
</div>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() || !isLiveTranscodingEnabled() }">
<label i18n for="liveTranscodingThreads">Live transcoding threads</label>
<div class="peertube-select-container">
<select id="liveTranscodingThreads" formControlName="threads" class="form-control">
<option *ngFor="let transcodingThreadOption of transcodingThreadOptions" [value]="transcodingThreadOption.value">
{{ transcodingThreadOption.label }}
</option>
</select>
</div>
<div *ngIf="formErrors.live.transcoding.threads" class="form-error">{{ formErrors.live.transcoding.threads }}</div>
</div>
<div class="form-group" [ngClass]="{ 'disabled-checkbox-extra': !isLiveEnabled() || !isLiveTranscodingEnabled() }">
<label i18n for="liveTranscodingThreads">Live resolutions to generate</label>
<div class="ml-2 mt-2 d-flex flex-column">
<ng-container formGroupName="resolutions">
<div class="form-group" *ngFor="let resolution of liveResolutions">
<my-peertube-checkbox
[inputName]="getResolutionKey(resolution.id)" [formControlName]="resolution.id"
labelText="{{resolution.label}}"
>
<ng-template *ngIf="resolution.description" ptTemplate="help">
<div [innerHTML]="resolution.description"></div>
</ng-template>
</my-peertube-checkbox>
</div>
</ng-container>
</div>
</div>
</ng-container>
</ng-container>
</my-peertube-checkbox>
</div>
@ -778,7 +802,7 @@
</div>
</ng-template>
</ng-container>
</div>
<ng-container ngbNavItem="advanced-configuration">
<a ngbNavLink i18n>Advanced configuration</a>
@ -1026,9 +1050,15 @@
<div class="form-row mt-4"> <!-- submit placement block -->
<div class="col-md-7 col-xl-5"></div>
<div class="col-md-5 col-xl-5">
<span class="form-error submit-error" i18n *ngIf="!form.valid">It seems like the configuration is invalid. Please search for potential errors in the different tabs.</span>
<span class="form-error submit-error" i18n *ngIf="!form.valid">
It seems like the configuration is invalid. Please search for potential errors in the different tabs.
</span>
<input (click)="formValidated()" type="submit" i18n-value value="Update configuration" [disabled]="!form.valid">
<span class="form-error submit-error" i18n *ngIf="!hasLiveAllowReplayConsistentOptions()">
You cannot allow live replay if you don't enable transcoding.
</span>
<input (click)="formValidated()" type="submit" i18n-value value="Update configuration" [disabled]="!form.valid || !hasConsistentOptions()">
</div>
</div>
</form>

30
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
View File

@ -36,6 +36,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
resolutions: { id: string, label: string, description?: string }[] = []
liveResolutions: { id: string, label: string, description?: string }[] = []
transcodingThreadOptions: { label: string, value: number }[] = []
liveMaxDurationOptions: { label: string, value: number }[] = []
languageItems: SelectOptionsItem[] = []
categoryItems: SelectOptionsItem[] = []
@ -92,6 +93,14 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
{ value: 4, label: '4' },
{ value: 8, label: '8' }
]
this.liveMaxDurationOptions = [
{ value: 0, label: $localize`No limit` },
{ value: 1000 * 3600, label: $localize`1 hour` },
{ value: 1000 * 3600 * 3, label: $localize`3 hours` },
{ value: 1000 * 3600 * 5, label: $localize`5 hours` },
{ value: 1000 * 3600 * 10, label: $localize`10 hours` }
]
}
get videoQuotaOptions () {
@ -114,7 +123,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
ngOnInit () {
this.serverConfig = this.serverService.getTmpConfig()
this.serverService.getConfig()
.subscribe(config => this.serverConfig = config)
.subscribe(config => {
this.serverConfig = config
})
const formGroupData: { [key in keyof CustomConfig ]: any } = {
instance: {
@ -204,6 +215,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
live: {
enabled: null,
maxDuration: null,
allowReplay: null,
transcoding: {
enabled: null,
threads: TRANSCODING_THREADS_VALIDATOR,
@ -341,6 +355,20 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
}
}
hasConsistentOptions () {
if (this.hasLiveAllowReplayConsistentOptions()) return true
return false
}
hasLiveAllowReplayConsistentOptions () {
if (this.isTranscodingEnabled() === false && this.isLiveEnabled() && this.form.value['live']['allowReplay'] === true) {
return false
}
return true
}
private updateForm () {
this.form.patchValue(this.customConfig)
}

2
client/src/app/core/server/server.service.ts
View File

@ -76,6 +76,8 @@ export class ServerService {
},
live: {
enabled: false,
allowReplay: true,
maxDuration: null,
transcoding: {
enabled: false,
enabledResolutions: []

2
config/test.yaml
View File

@ -89,7 +89,7 @@ live:
port: 1935
transcoding:
enabled: true
enabled: false
threads: 2
resolutions:

5
server/controllers/api/config.ts
View File

@ -118,6 +118,9 @@ async function getConfig (req: express.Request, res: express.Response) {
live: {
enabled: CONFIG.LIVE.ENABLED,
allowReplay: CONFIG.LIVE.ALLOW_REPLAY,
maxDuration: CONFIG.LIVE.MAX_DURATION,
transcoding: {
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
enabledResolutions: getEnabledResolutions('live')
@ -425,6 +428,8 @@ function customConfig (): CustomConfig {
},
live: {
enabled: CONFIG.LIVE.ENABLED,
allowReplay: CONFIG.LIVE.ALLOW_REPLAY,
maxDuration: CONFIG.LIVE.MAX_DURATION,
transcoding: {
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
threads: CONFIG.LIVE.TRANSCODING.THREADS,

6
server/controllers/api/users/me.ts
View File

@ -9,7 +9,7 @@ import { MIMETYPES } from '../../../initializers/constants'
import { sequelizeTypescript } from '../../../initializers/database'
import { sendUpdateActor } from '../../../lib/activitypub/send'
import { updateActorAvatarFile } from '../../../lib/avatar'
import { sendVerifyUserEmail } from '../../../lib/user'
import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@ -133,8 +133,8 @@ async function getUserInformation (req: express.Request, res: express.Response)
async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) {
const user = res.locals.oauth.token.user
const videoQuotaUsed = await UserModel.getOriginalVideoFileTotalFromUser(user)
const videoQuotaUsedDaily = await UserModel.getOriginalVideoFileTotalDailyFromUser(user)
const videoQuotaUsed = await getOriginalVideoFileTotalFromUser(user)
const videoQuotaUsedDaily = await getOriginalVideoFileTotalDailyFromUser(user)
const data: UserVideoQuota = {
videoQuotaUsed,

1
server/helpers/core-utils.ts
View File

@ -41,6 +41,7 @@ const timeTable = {
}
export function parseDurationToMs (duration: number | string): number {
if (duration === null) return null
if (typeof duration === 'number') return duration
if (typeof duration === 'string') {

5
server/helpers/custom-validators/misc.ts
View File

@ -45,6 +45,10 @@ function isBooleanValid (value: any) {
return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value))
}
function isIntOrNull (value: any) {
return value === null || validator.isInt('' + value)
}
function toIntOrNull (value: string) {
const v = toValueOrNull(value)
@ -116,6 +120,7 @@ export {
isArrayOf,
isNotEmptyIntArray,
isArray,
isIntOrNull,
isIdValid,
isSafePath,
isUUIDValid,

22
server/helpers/ffmpeg-utils.ts
View File

@ -5,7 +5,7 @@ import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
import { checkFFmpegEncoders } from '../initializers/checker-before-init'
import { CONFIG } from '../initializers/config'
import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
import { processImage } from './image-utils'
import { logger } from './logger'
@ -353,7 +353,7 @@ function convertWebPToJPG (path: string, destination: string): Promise<void> {
})
}
function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[]) {
function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[], deleteSegments: boolean) {
const command = getFFmpeg(rtmpUrl)
command.inputOption('-fflags nobuffer')
@ -399,7 +399,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb
varStreamMap.push(`v:${i},a:${i}`)
}
addDefaultLiveHLSParams(command, outPath)
addDefaultLiveHLSParams(command, outPath, deleteSegments)
command.outputOption('-var_stream_map', varStreamMap.join(' '))
@ -408,7 +408,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb
return command
}
function runLiveMuxing (rtmpUrl: string, outPath: string) {
function runLiveMuxing (rtmpUrl: string, outPath: string, deleteSegments: boolean) {
const command = getFFmpeg(rtmpUrl)
command.inputOption('-fflags nobuffer')
@ -417,7 +417,7 @@ function runLiveMuxing (rtmpUrl: string, outPath: string) {
command.outputOption('-map 0:a?')
command.outputOption('-map 0:v?')
addDefaultLiveHLSParams(command, outPath)
addDefaultLiveHLSParams(command, outPath, deleteSegments)
command.run()
@ -457,10 +457,14 @@ function addDefaultX264Params (command: ffmpeg.FfmpegCommand) {
.outputOption('-map_metadata -1') // strip all metadata
}
function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
command.outputOption('-hls_time 4')
command.outputOption('-hls_list_size 15')
command.outputOption('-hls_flags delete_segments')
function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, deleteSegments: boolean) {
command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME)
command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
if (deleteSegments === true) {
command.outputOption('-hls_flags delete_segments')
}
command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%d.ts')}`)
command.outputOption('-master_pl_name master.m3u8')
command.outputOption(`-f hls`)

7
server/initializers/checker-after-init.ts
View File

@ -135,6 +135,13 @@ function checkConfig () {
}
}
// Live
if (CONFIG.LIVE.ENABLED === true) {
if (CONFIG.LIVE.ALLOW_REPLAY === true && CONFIG.TRANSCODING.ENABLED === false) {
return 'Live allow replay cannot be enabled if transcoding is not enabled.'
}
}
return null
}

7
server/initializers/checker-before-init.ts
View File

@ -37,8 +37,13 @@ function checkMissedConfig () {
'remote_redundancy.videos.accept_from',
'federation.videos.federate_unlisted',
'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url',
'search.search_index.disable_local_search', 'search.search_index.is_default_search'
'search.search_index.disable_local_search', 'search.search_index.is_default_search',
'live.enabled', 'live.allow_replay', 'live.max_duration',
'live.transcoding.enabled', 'live.transcoding.threads',
'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', 'live.transcoding.resolutions.480p',
'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', 'live.transcoding.resolutions.2160p'
]
const requiredAlternatives = [
[ // set
[ 'redis.hostname', 'redis.port' ], // alternative

3
server/initializers/config.ts
View File

@ -201,6 +201,9 @@ const CONFIG = {
LIVE: {
get ENABLED () { return config.get<boolean>('live.enabled') },
get MAX_DURATION () { return parseDurationToMs(config.get<string>('live.max_duration')) },
get ALLOW_REPLAY () { return config.get<boolean>('live.allow_replay') },
RTMP: {
get PORT () { return config.get<number>('live.rtmp.port') }
},

7
server/initializers/constants.ts
View File

@ -608,7 +608,9 @@ const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
const VIDEO_LIVE = {
EXTENSION: '.ts',
CLEANUP_DELAY: 1000 * 60 * 5, // 5 mintues
CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes
SEGMENT_TIME: 4, // 4 seconds
SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist
RTMP: {
CHUNK_SIZE: 60000,
GOP_CACHE: true,
@ -620,7 +622,8 @@ const VIDEO_LIVE = {
const MEMOIZE_TTL = {
OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours
INFO_HASH_EXISTS: 1000 * 3600 * 12 // 12 hours
INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours
LIVE_ABLE_TO_UPLOAD: 1000 * 60 // 1 minute
}
const MEMOIZE_LENGTH = {

2
server/initializers/migrations/0535-video-live.ts
View File

@ -9,7 +9,7 @@ async function up (utils: {
const query = `
CREATE TABLE IF NOT EXISTS "videoLive" (
"id" SERIAL ,
"streamKey" VARCHAR(255) NOT NULL,
"streamKey" VARCHAR(255),
"videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,

3
server/lib/job-queue/handlers/video-import.ts
View File

@ -4,6 +4,7 @@ import { extname } from 'path'
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
import { isPostImportVideoAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { isAbleToUploadVideo } from '@server/lib/user'
import { getVideoFilePath } from '@server/lib/video-paths'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
import {
@ -108,7 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
// Get information about this video
const stats = await stat(tempVideoPath)
const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size })
const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size)
if (isAble === false) {
throw new Error('The user video quota is exceeded with this video to import.')
}

89
server/lib/live-manager.ts
View File

@ -2,24 +2,27 @@
import { AsyncQueue, queue } from 'async'
import * as chokidar from 'chokidar'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { ensureDir } from 'fs-extra'
import { ensureDir, stat } from 'fs-extra'
import { basename } from 'path'
import { computeResolutionsToTranscode, runLiveMuxing, runLiveTranscoding } from '@server/helpers/ffmpeg-utils'
import { logger } from '@server/helpers/logger'
import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants'
import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants'
import { UserModel } from '@server/models/account/user'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { MStreamingPlaylist, MVideoLiveVideo } from '@server/types/models'
import { MStreamingPlaylist, MUser, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models'
import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
import { federateVideoIfNeeded } from './activitypub/videos'
import { buildSha256Segment } from './hls'
import { JobQueue } from './job-queue'
import { PeerTubeSocket } from './peertube-socket'
import { isAbleToUploadVideo } from './user'
import { getHLSDirectory } from './video-paths'
import memoizee = require('memoizee')
const NodeRtmpServer = require('node-media-server/node_rtmp_server')
const context = require('node-media-server/node_core_ctx')
const nodeMediaServerLogger = require('node-media-server/node_core_logger')
@ -53,6 +56,11 @@ class LiveManager {
private readonly transSessions = new Map<string, FfmpegCommand>()
private readonly videoSessions = new Map<number, string>()
private readonly segmentsSha256 = new Map<string, Map<string, string>>()
private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>()
private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
return isAbleToUploadVideo(userId, 1000)
}, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
private segmentsSha256Queue: AsyncQueue<SegmentSha256QueueParam>
private rtmpServer: any
@ -127,7 +135,7 @@ class LiveManager {
this.abortSession(sessionId)
this.onEndTransmuxing(videoId)
this.onEndTransmuxing(videoId, true)
.catch(err => logger.error('Cannot end transmuxing of video %d.', videoId, { err }))
}
@ -196,8 +204,18 @@ class LiveManager {
originalResolution: number
}) {
const { sessionId, videoLive, playlist, streamPath, resolutionsEnabled, originalResolution } = options
const startStreamDateTime = new Date().getTime()
const allResolutions = resolutionsEnabled.concat([ originalResolution ])
const user = await UserModel.loadByLiveId(videoLive.id)
if (!this.livesPerUser.has(user.id)) {
this.livesPerUser.set(user.id, [])
}
const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 }
const livesOfUser = this.livesPerUser.get(user.id)
livesOfUser.push(currentUserLive)
for (let i = 0; i < allResolutions.length; i++) {
const resolution = allResolutions[i]
@ -216,26 +234,47 @@ class LiveManager {
const outPath = getHLSDirectory(videoLive.Video)
await ensureDir(outPath)
const deleteSegments = videoLive.saveReplay === false
const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED
? runLiveTranscoding(rtmpUrl, outPath, allResolutions)
: runLiveMuxing(rtmpUrl, outPath)
? runLiveTranscoding(rtmpUrl, outPath, allResolutions, deleteSegments)
: runLiveMuxing(rtmpUrl, outPath, deleteSegments)
logger.info('Running live muxing/transcoding.')
this.transSessions.set(sessionId, ffmpegExec)
const videoUUID = videoLive.Video.uuid
const tsWatcher = chokidar.watch(outPath + '/*.ts')
const updateHandler = segmentPath => {
this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID })
const updateSegment = segmentPath => this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID })
const addHandler = segmentPath => {
updateSegment(segmentPath)
if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
this.stopSessionOf(videoLive.videoId)
}
if (videoLive.saveReplay === true) {
stat(segmentPath)
.then(segmentStat => {
currentUserLive.size += segmentStat.size
})
.then(() => this.isQuotaConstraintValid(user, videoLive))
.then(quotaValid => {
if (quotaValid !== true) {
this.stopSessionOf(videoLive.videoId)
}
})
.catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err }))
}
}
const deleteHandler = segmentPath => this.segmentsSha256Queue.push({ operation: 'delete', segmentPath, videoUUID })
tsWatcher.on('add', p => updateHandler(p))
tsWatcher.on('change', p => updateHandler(p))
tsWatcher.on('add', p => addHandler(p))
tsWatcher.on('change', p => updateSegment(p))
tsWatcher.on('unlink', p => deleteHandler(p))
const masterWatcher = chokidar.watch(outPath + '/master.m3u8')
@ -280,7 +319,14 @@ class LiveManager {
ffmpegExec.on('end', () => onFFmpegEnded())
}
private async onEndTransmuxing (videoId: number) {
getLiveQuotaUsedByUser (userId: number) {
const currentLives = this.livesPerUser.get(userId)
if (!currentLives) return 0
return currentLives.reduce((sum, obj) => sum + obj.size, 0)
}
private async onEndTransmuxing (videoId: number, cleanupNow = false) {
try {
const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
if (!fullVideo) return
@ -290,7 +336,7 @@ class LiveManager {
payload: {
videoId: fullVideo.id
}
}, { delay: VIDEO_LIVE.CLEANUP_DELAY })
}, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
// FIXME: use end
fullVideo.state = VideoState.WAITING_FOR_LIVE
@ -337,6 +383,23 @@ class LiveManager {
filesMap.delete(segmentName)
}
private isDurationConstraintValid (streamingStartTime: number) {
const maxDuration = CONFIG.LIVE.MAX_DURATION
// No limit
if (maxDuration === null) return true
const now = new Date().getTime()
const max = streamingStartTime + maxDuration
return now <= max
}
private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) {
if (live.saveReplay !== true) return true
return this.isAbleToUploadVideoWithCache(user.id)
}
static get Instance () {
return this.instance || (this.instance = new this())
}

80
server/lib/user.ts
View File

@ -1,20 +1,24 @@
import { v4 as uuidv4 } from 'uuid'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants'
import { AccountModel } from '../models/account/account'
import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor'
import { createLocalVideoChannel } from './video-channel'
import { ActorModel } from '../models/activitypub/actor'
import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
import { createWatchLaterPlaylist } from './video-playlist'
import { sequelizeTypescript } from '../initializers/database'
import { Transaction } from 'sequelize/types'
import { Redis } from './redis'
import { Emailer } from './emailer'
import { v4 as uuidv4 } from 'uuid'
import { UserModel } from '@server/models/account/user'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants'
import { sequelizeTypescript } from '../initializers/database'
import { AccountModel } from '../models/account/account'
import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
import { ActorModel } from '../models/activitypub/actor'
import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models'
import { MUser, MUserDefault, MUserId } from '../types/models/user'
import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor'
import { getAccountActivityPubUrl } from './activitypub/url'
import { Emailer } from './emailer'
import { LiveManager } from './live-manager'
import { Redis } from './redis'
import { createLocalVideoChannel } from './video-channel'
import { createWatchLaterPlaylist } from './video-playlist'
import memoizee = require('memoizee')
type ChannelNames = { name: string, displayName: string }
@ -116,13 +120,61 @@ async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) {
await Emailer.Instance.addVerifyEmailJob(username, email, url)
}
async function getOriginalVideoFileTotalFromUser (user: MUserId) {
// Don't use sequelize because we need to use a sub query
const query = UserModel.generateUserQuotaBaseSQL({
withSelect: true,
whereUserId: '$userId'
})
const base = await UserModel.getTotalRawQuery(query, user.id)
return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id)
}
// Returns cumulative size of all video files uploaded in the last 24 hours.
async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) {
// Don't use sequelize because we need to use a sub query
const query = UserModel.generateUserQuotaBaseSQL({
withSelect: true,
whereUserId: '$userId',
where: '"video"."createdAt" > now() - interval \'24 hours\''
})
const base = await UserModel.getTotalRawQuery(query, user.id)
return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id)
}
async function isAbleToUploadVideo (userId: number, size: number) {
const user = await UserModel.loadById(userId)
if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true)
const [ totalBytes, totalBytesDaily ] = await Promise.all([
getOriginalVideoFileTotalFromUser(user.id),
getOriginalVideoFileTotalDailyFromUser(user.id)
])
const uploadedTotal = size + totalBytes
const uploadedDaily = size + totalBytesDaily
if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota
if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily
return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily
}
// ---------------------------------------------------------------------------
export {
getOriginalVideoFileTotalFromUser,
getOriginalVideoFileTotalDailyFromUser,
createApplicationActor,
createUserAccountAndChannelAndPlaylist,
createLocalAccountWithoutKeys,
sendVerifyUserEmail
sendVerifyUserEmail,
isAbleToUploadVideo
}
// ---------------------------------------------------------------------------

40
server/middlewares/validators/config.ts
View File

@ -1,12 +1,13 @@
import * as express from 'express'
import { body } from 'express-validator'
import { isIntOrNull } from '@server/helpers/custom-validators/misc'
import { isEmailEnabled } from '@server/initializers/config'
import { CustomConfig } from '../../../shared/models/server/custom-config.model'
import { isThemeNameValid } from '../../helpers/custom-validators/plugins'
import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users'
import { logger } from '../../helpers/logger'
import { CustomConfig } from '../../../shared/models/server/custom-config.model'
import { areValidationErrors } from './utils'
import { isThemeNameValid } from '../../helpers/custom-validators/plugins'
import { isThemeRegistered } from '../../lib/plugins/theme-utils'
import { isEmailEnabled } from '@server/initializers/config'
import { areValidationErrors } from './utils'
const customConfigUpdateValidator = [
body('instance.name').exists().withMessage('Should have a valid instance name'),
@ -43,6 +44,7 @@ const customConfigUpdateValidator = [
body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),
body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
body('transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'),
body('transcoding.webtorrent.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'),
body('transcoding.hls.enabled').isBoolean().withMessage('Should have a valid hls transcoding enabled boolean'),
@ -60,6 +62,18 @@ const customConfigUpdateValidator = [
body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'),
body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'),
body('live.enabled').isBoolean().withMessage('Should have a valid live enabled boolean'),
body('live.allowReplay').isBoolean().withMessage('Should have a valid live allow replay boolean'),
body('live.maxDuration').custom(isIntOrNull).withMessage('Should have a valid live max duration'),
body('live.transcoding.enabled').isBoolean().withMessage('Should have a valid live transcoding enabled boolean'),
body('live.transcoding.threads').isInt().withMessage('Should have a valid live transcoding threads'),
body('live.transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'),
body('live.transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'),
body('live.transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),
body('live.transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
body('live.transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
body('live.transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'),
body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'),
body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'),
body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'),
@ -71,8 +85,9 @@ const customConfigUpdateValidator = [
logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!checkInvalidConfigIfEmailDisabled(req.body as CustomConfig, res)) return
if (!checkInvalidTranscodingConfig(req.body as CustomConfig, res)) return
if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
if (!checkInvalidTranscodingConfig(req.body, res)) return
if (!checkInvalidLiveConfig(req.body, res)) return
return next()
}
@ -109,3 +124,16 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
return true
}
function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
if (customConfig.live.enabled === false) return true
if (customConfig.live.allowReplay === true && customConfig.transcoding.enabled === false) {
res.status(400)
.send({ error: 'You cannot allow live replay if transcoding is not enabled' })
.end()
return false
}
return true
}

2
server/middlewares/validators/users.ts
View File

@ -497,7 +497,7 @@ export {
function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
const id = parseInt(idArg + '', 10)
return checkUserExist(() => UserModel.loadById(id, withStats), res)
return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
}
function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {

5
server/middlewares/validators/videos/videos.ts
View File

@ -1,5 +1,6 @@
import * as express from 'express'
import { body, param, query, ValidationChain } from 'express-validator'
import { isAbleToUploadVideo } from '@server/lib/user'
import { getServerActor } from '@server/models/application/application'
import { MVideoFullLight } from '@server/types/models'
import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared'
@ -73,7 +74,7 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([
if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
if (await user.isAbleToUploadVideo(videoFile) === false) {
if (await isAbleToUploadVideo(user.id, videoFile.size) === false) {
res.status(403)
.json({ error: 'The user video quota is exceeded with this video.' })
@ -291,7 +292,7 @@ const videosAcceptChangeOwnershipValidator = [
const user = res.locals.oauth.token.User
const videoChangeOwnership = res.locals.videoChangeOwnership
const isAble = await user.isAbleToUploadVideo(videoChangeOwnership.Video.getMaxQualityFile())
const isAble = await isAbleToUploadVideo(user.id, videoChangeOwnership.Video.getMaxQualityFile().size)
if (isAble === false) {
res.status(403)
.json({ error: 'The user video quota is exceeded with this video.' })

159
server/models/account/user.ts
View File

@ -23,6 +23,7 @@ import {
} from 'sequelize-typescript'
import {
MMyUserFormattable,
MUser,
MUserDefault,
MUserFormattable,
MUserId,
@ -70,6 +71,7 @@ import { VideoImportModel } from '../video/video-import'
import { VideoPlaylistModel } from '../video/video-playlist'
import { AccountModel } from './account'
import { UserNotificationSettingModel } from './user-notification-setting'
import { VideoLiveModel } from '../video/video-live'
enum ScopeNames {
FOR_ME_API = 'FOR_ME_API',
@ -540,7 +542,11 @@ export class UserModel extends Model<UserModel> {
return UserModel.findAll(query)
}
static loadById (id: number, withStats = false): Bluebird<MUserDefault> {
static loadById (id: number): Bluebird<MUser> {
return UserModel.unscoped().findByPk(id)
}
static loadByIdWithChannels (id: number, withStats = false): Bluebird<MUserDefault> {
const scopes = [
ScopeNames.WITH_VIDEOCHANNELS
]
@ -685,26 +691,85 @@ export class UserModel extends Model<UserModel> {
return UserModel.findOne(query)
}
static getOriginalVideoFileTotalFromUser (user: MUserId) {
// Don't use sequelize because we need to use a sub query
const query = UserModel.generateUserQuotaBaseSQL({
withSelect: true,
whereUserId: '$userId'
})
static loadByLiveId (liveId: number): Bluebird<MUser> {
const query = {
include: [
{
attributes: [ 'id' ],
model: AccountModel.unscoped(),
required: true,
include: [
{
attributes: [ 'id' ],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [ 'id' ],
model: VideoModel.unscoped(),
required: true,
include: [
{
attributes: [ 'id', 'videoId' ],
model: VideoLiveModel.unscoped(),
required: true,
where: {
id: liveId
}
}
]
}
]
}
]
}
]
}
return UserModel.getTotalRawQuery(query, user.id)
return UserModel.findOne(query)
}
// Returns cumulative size of all video files uploaded in the last 24 hours.
static getOriginalVideoFileTotalDailyFromUser (user: MUserId) {
// Don't use sequelize because we need to use a sub query
const query = UserModel.generateUserQuotaBaseSQL({
withSelect: true,
whereUserId: '$userId',
where: '"video"."createdAt" > now() - interval \'24 hours\''
})
static generateUserQuotaBaseSQL (options: {
whereUserId: '$userId' | '"UserModel"."id"'
withSelect: boolean
where?: string
}) {
const andWhere = options.where
? 'AND ' + options.where
: ''
return UserModel.getTotalRawQuery(query, user.id)
const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' +
'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' +
`WHERE "account"."userId" = ${options.whereUserId} ${andWhere}`
const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' +
'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' +
videoChannelJoin
const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' +
'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' +
'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' +
videoChannelJoin
return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' +
'FROM (' +
`SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` +
'GROUP BY "t1"."videoId"' +
') t2'
}
static getTotalRawQuery (query: string, userId: number) {
const options = {
bind: { userId },
type: QueryTypes.SELECT as QueryTypes.SELECT
}
return UserModel.sequelize.query<{ total: string }>(query, options)
.then(([ { total } ]) => {
if (total === null) return 0
return parseInt(total, 10)
})
}
static async getStats () {
@ -874,64 +939,4 @@ export class UserModel extends Model<UserModel> {
return Object.assign(formatted, { specialPlaylists })
}
async isAbleToUploadVideo (videoFile: { size: number }) {
if (this.videoQuota === -1 && this.videoQuotaDaily === -1) return Promise.resolve(true)
const [ totalBytes, totalBytesDaily ] = await Promise.all([
UserModel.getOriginalVideoFileTotalFromUser(this),
UserModel.getOriginalVideoFileTotalDailyFromUser(this)
])
const uploadedTotal = videoFile.size + totalBytes
const uploadedDaily = videoFile.size + totalBytesDaily
if (this.videoQuotaDaily === -1) return uploadedTotal < this.videoQuota
if (this.videoQuota === -1) return uploadedDaily < this.videoQuotaDaily
return uploadedTotal < this.videoQuota && uploadedDaily < this.videoQuotaDaily
}
private static generateUserQuotaBaseSQL (options: {
whereUserId: '$userId' | '"UserModel"."id"'
withSelect: boolean
where?: string
}) {
const andWhere = options.where
? 'AND ' + options.where
: ''
const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' +
'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' +
`WHERE "account"."userId" = ${options.whereUserId} ${andWhere}`
const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' +
'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' +
videoChannelJoin
const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' +
'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' +
'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' +
videoChannelJoin
return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' +
'FROM (' +
`SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` +
'GROUP BY "t1"."videoId"' +
') t2'
}
private static getTotalRawQuery (query: string, userId: number) {
const options = {
bind: { userId },
type: QueryTypes.SELECT as QueryTypes.SELECT
}
return UserModel.sequelize.query<{ total: string }>(query, options)
.then(([ { total } ]) => {
if (total === null) return 0
return parseInt(total, 10)
})
}
}

10
server/models/video/video-live.ts
View File

@ -30,10 +30,18 @@ import { VideoBlacklistModel } from './video-blacklist'
})
export class VideoLiveModel extends Model<VideoLiveModel> {
@AllowNull(false)
@AllowNull(true)
@Column(DataType.STRING)
streamKey: string
@AllowNull(false)
@Column
perpetualLive: boolean
@AllowNull(false)
@Column
saveReplay: boolean
@CreatedAt
createdAt: Date

3
server/tests/api/check-params/config.ts
View File

@ -103,6 +103,9 @@ describe('Test config API validators', function () {
live: {
enabled: true,
allowReplay: false,
maxDuration: null,
transcoding: {
enabled: true,
threads: 4,

6
server/tests/api/server/config.ts
View File

@ -79,6 +79,8 @@ function checkInitialConfig (server: ServerInfo, data: CustomConfig) {
expect(data.transcoding.hls.enabled).to.be.true
expect(data.live.enabled).to.be.false
expect(data.live.allowReplay).to.be.true
expect(data.live.maxDuration).to.equal(1000 * 3600 * 5)
expect(data.live.transcoding.enabled).to.be.false
expect(data.live.transcoding.threads).to.equal(2)
expect(data.live.transcoding.resolutions['240p']).to.be.false
@ -162,6 +164,8 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.transcoding.webtorrent.enabled).to.be.true
expect(data.live.enabled).to.be.true
expect(data.live.allowReplay).to.be.false
expect(data.live.maxDuration).to.equal(5000)
expect(data.live.transcoding.enabled).to.be.true
expect(data.live.transcoding.threads).to.equal(4)
expect(data.live.transcoding.resolutions['240p']).to.be.true
@ -324,6 +328,8 @@ describe('Test config', function () {
},
live: {
enabled: true,
allowReplay: false,
maxDuration: 5000,
transcoding: {
enabled: true,
threads: 4,

2
shared/extra-utils/server/config.ts
View File

@ -128,6 +128,8 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti
},
live: {
enabled: true,
allowReplay: false,
maxDuration: null,
transcoding: {
enabled: true,
threads: 4,

3
shared/models/server/custom-config.model.ts
View File

@ -98,6 +98,9 @@ export interface CustomConfig {
live: {
enabled: boolean
allowReplay: boolean
maxDuration: number
transcoding: {
enabled: boolean
threads: number

3
shared/models/server/server-config.model.ts
View File

@ -101,6 +101,9 @@ export interface ServerConfig {
live: {
enabled: boolean
maxDuration: number
allowReplay: boolean
transcoding: {
enabled: boolean