PeerTube/server/models/video/video.ts

1781 lines
47 KiB
TypeScript
Raw Normal View History

2021-08-27 07:32:44 -05:00
import Bluebird from 'bluebird'
2020-07-01 09:05:30 -05:00
import { remove } from 'fs-extra'
2021-06-10 01:53:32 -05:00
import { maxBy, minBy } from 'lodash'
import { join } from 'path'
2020-12-08 07:30:29 -06:00
import { FindOptions, Includeable, IncludeOptions, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize'
2017-12-12 10:53:50 -06:00
import {
AllowNull,
BeforeDestroy,
BelongsTo,
BelongsToMany,
Column,
CreatedAt,
DataType,
Default,
ForeignKey,
HasMany,
HasOne,
Is,
IsInt,
IsUUID,
Min,
Model,
Scopes,
Table,
2018-08-31 10:18:13 -05:00
UpdatedAt
2017-12-12 10:53:50 -06:00
} from 'sequelize-typescript'
2020-07-01 09:05:30 -05:00
import { buildNSFWFilter } from '@server/helpers/express-utils'
import { uuidToShort } from '@server/helpers/uuid'
2020-11-04 07:16:57 -06:00
import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
2021-06-16 08:14:41 -05:00
import { LiveManager } from '@server/lib/live/live-manager'
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
2020-07-01 09:05:30 -05:00
import { getServerActor } from '@server/models/application/application'
import { ModelCache } from '@server/models/model-cache'
2021-07-29 07:17:03 -05:00
import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
import { VideoInclude } from '@shared/models'
2020-07-01 09:05:30 -05:00
import { VideoFile } from '@shared/models/videos/video-file.model'
import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
2020-09-17 06:59:02 -05:00
import { VideoObject } from '../../../shared/models/activitypub/objects'
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
2020-07-01 09:05:30 -05:00
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
2019-07-15 02:22:57 -05:00
import { peertubeTruncate } from '../../helpers/core-utils'
2017-12-28 04:16:08 -06:00
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
2021-10-20 03:04:06 -05:00
import { exists, isBooleanValid } from '../../helpers/custom-validators/misc'
2017-12-12 10:53:50 -06:00
import {
isVideoDescriptionValid,
isVideoDurationValid,
2019-02-26 03:55:40 -06:00
isVideoNameValid,
isVideoPrivacyValid,
isVideoStateValid,
2018-03-12 05:06:15 -05:00
isVideoSupportValid
2017-12-12 10:53:50 -06:00
} from '../../helpers/custom-validators/videos'
2020-11-20 10:16:55 -06:00
import { getVideoFileResolution } from '../../helpers/ffprobe-utils'
2017-12-28 04:16:08 -06:00
import { logger } from '../../helpers/logger'
2020-07-01 09:05:30 -05:00
import { CONFIG } from '../../initializers/config'
2021-06-11 07:45:58 -05:00
import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
2017-12-14 10:38:41 -06:00
import { sendDeleteVideo } from '../../lib/activitypub/send'
2019-08-15 04:53:26 -05:00
import {
MChannel,
2019-08-20 06:52:49 -05:00
MChannelAccountDefault,
2019-08-15 04:53:26 -05:00
MChannelId,
MStreamingPlaylist,
MStreamingPlaylistFilesVideo,
2019-08-15 04:53:26 -05:00
MUserAccountId,
MUserId,
MVideo,
2019-08-15 04:53:26 -05:00
MVideoAccountLight,
2019-08-20 06:52:49 -05:00
MVideoAccountLightBlacklistAllFiles,
2019-08-21 07:31:57 -05:00
MVideoAP,
2019-08-15 04:53:26 -05:00
MVideoDetails,
MVideoFileVideo,
2019-08-21 07:31:57 -05:00
MVideoFormattable,
MVideoFormattableDetails,
2019-08-20 06:52:49 -05:00
MVideoForUser,
2019-08-15 04:53:26 -05:00
MVideoFullLight,
2021-06-11 07:09:33 -05:00
MVideoId,
2020-03-05 08:04:57 -06:00
MVideoImmutable,
2019-08-15 04:53:26 -05:00
MVideoThumbnail,
2019-08-22 03:46:54 -05:00
MVideoThumbnailBlacklist,
2019-08-21 07:31:57 -05:00
MVideoWithAllFiles,
2021-06-11 07:09:33 -05:00
MVideoWithFile
2020-06-18 03:45:25 -05:00
} from '../../types/models'
import { MThumbnail } from '../../types/models/video/thumbnail'
import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../types/models/video/video-file'
2020-07-01 09:05:30 -05:00
import { VideoAbuseModel } from '../abuse/video-abuse'
import { AccountModel } from '../account/account'
import { AccountVideoRateModel } from '../account/account-video-rate'
2021-05-11 04:15:29 -05:00
import { ActorModel } from '../actor/actor'
import { ActorImageModel } from '../actor/actor-image'
2020-07-01 09:05:30 -05:00
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { ServerModel } from '../server/server'
2021-02-18 03:15:11 -06:00
import { TrackerModel } from '../server/tracker'
import { VideoTrackerModel } from '../server/video-tracker'
import { setAsUpdated } from '../shared'
2021-05-11 04:15:29 -05:00
import { UserModel } from '../user/user'
import { UserVideoHistoryModel } from '../user/user-video-history'
2020-07-01 09:05:30 -05:00
import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils'
2021-06-10 01:53:32 -05:00
import {
videoFilesModelToFormattedJSON,
VideoFormattingJSONOptions,
videoModelToActivityPubObject,
videoModelToFormattedDetailsJSON,
videoModelToFormattedJSON
} from './formatter/video-format-utils'
2020-07-01 09:05:30 -05:00
import { ScheduleVideoUpdateModel } from './schedule-video-update'
import { VideoModelGetQueryBuilder } from './sql/video-model-get-query-builder'
import { BuildVideosListQueryOptions, DisplayOnlyForFollowerOptions, VideosIdListQueryBuilder } from './sql/videos-id-list-query-builder'
2021-06-10 01:53:32 -05:00
import { VideosModelListQueryBuilder } from './sql/videos-model-list-query-builder'
2020-07-01 09:05:30 -05:00
import { TagModel } from './tag'
import { ThumbnailModel } from './thumbnail'
import { VideoBlacklistModel } from './video-blacklist'
import { VideoCaptionModel } from './video-caption'
import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from './video-channel'
import { VideoCommentModel } from './video-comment'
import { VideoFileModel } from './video-file'
import { VideoImportModel } from './video-import'
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
import { VideoJobInfoModel } from './video-job-info'
2020-11-02 08:43:44 -06:00
import { VideoLiveModel } from './video-live'
2020-07-01 09:05:30 -05:00
import { VideoPlaylistElementModel } from './video-playlist-element'
import { VideoShareModel } from './video-share'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
import { VideoTagModel } from './video-tag'
import { VideoViewModel } from './video-view'
2018-10-05 04:15:06 -05:00
export enum ScopeNames {
FOR_API = 'FOR_API',
2018-01-04 04:19:16 -06:00
WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS',
2017-12-14 03:07:57 -06:00
WITH_TAGS = 'WITH_TAGS',
WITH_WEBTORRENT_FILES = 'WITH_WEBTORRENT_FILES',
2018-08-14 02:08:47 -05:00
WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
2018-10-05 04:15:06 -05:00
WITH_BLACKLISTED = 'WITH_BLACKLISTED',
2019-01-29 01:37:25 -06:00
WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES',
2021-06-11 07:45:58 -05:00
WITH_USER_HISTORY = 'WITH_USER_HISTORY',
WITH_THUMBNAILS = 'WITH_THUMBNAILS'
2017-12-14 03:07:57 -06:00
}
2019-07-31 08:57:32 -05:00
export type ForAPIOptions = {
ids?: number[]
2019-02-26 03:55:40 -06:00
videoPlaylistId?: number
2019-07-31 08:57:32 -05:00
withAccountBlockerIds?: number[]
}
2019-04-23 02:50:57 -05:00
@Scopes(() => ({
[ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: {
attributes: [ 'id', 'url', 'uuid', 'remote' ]
},
2020-01-31 09:56:52 -06:00
[ScopeNames.FOR_API]: (options: ForAPIOptions) => {
2020-12-08 07:30:29 -06:00
const include: Includeable[] = [
{
model: VideoChannelModel.scope({
method: [
VideoChannelScopeNames.SUMMARY, {
withAccount: true,
withAccountBlockerIds: options.withAccountBlockerIds
} as SummaryOptions
]
}),
required: true
},
{
attributes: [ 'type', 'filename' ],
model: ThumbnailModel,
required: false
}
]
const query: FindOptions = {}
2019-07-31 08:57:32 -05:00
if (options.ids) {
query.where = {
id: {
2020-01-31 09:56:52 -06:00
[Op.in]: options.ids
2019-07-31 08:57:32 -05:00
}
}
}
2019-02-26 03:55:40 -06:00
if (options.videoPlaylistId) {
2020-12-08 07:30:29 -06:00
include.push({
2019-02-26 03:55:40 -06:00
model: VideoPlaylistElementModel.unscoped(),
2019-03-12 05:40:42 -05:00
required: true,
where: {
videoPlaylistId: options.videoPlaylistId
}
2019-02-26 03:55:40 -06:00
})
}
2020-12-08 07:30:29 -06:00
query.include = include
return query
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_THUMBNAILS]: {
include: [
{
2019-04-23 02:50:57 -05:00
model: ThumbnailModel,
required: false
}
]
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_ACCOUNT_DETAILS]: {
2017-12-14 03:07:57 -06:00
include: [
{
2019-04-23 02:50:57 -05:00
model: VideoChannelModel.unscoped(),
2017-12-14 03:07:57 -06:00
required: true,
include: [
2018-01-11 07:30:27 -06:00
{
attributes: {
exclude: [ 'privateKey', 'publicKey' ]
},
2019-04-23 02:50:57 -05:00
model: ActorModel.unscoped(),
2018-01-18 10:44:04 -06:00
required: true,
include: [
{
attributes: [ 'host' ],
2019-04-23 02:50:57 -05:00
model: ServerModel.unscoped(),
2018-01-18 10:44:04 -06:00
required: false
},
{
2021-04-06 04:35:56 -05:00
model: ActorImageModel.unscoped(),
as: 'Avatar',
required: false
2018-01-18 10:44:04 -06:00
}
]
2018-01-11 07:30:27 -06:00
},
2017-12-14 03:07:57 -06:00
{
2019-04-23 02:50:57 -05:00
model: AccountModel.unscoped(),
2017-12-14 03:07:57 -06:00
required: true,
include: [
{
2019-04-23 02:50:57 -05:00
model: ActorModel.unscoped(),
2018-01-11 07:30:27 -06:00
attributes: {
exclude: [ 'privateKey', 'publicKey' ]
},
2017-12-14 10:38:41 -06:00
required: true,
include: [
{
2018-01-18 10:44:04 -06:00
attributes: [ 'host' ],
2019-04-23 02:50:57 -05:00
model: ServerModel.unscoped(),
2017-12-14 10:38:41 -06:00
required: false
2018-02-16 04:19:54 -06:00
},
{
2021-04-06 04:35:56 -05:00
model: ActorImageModel.unscoped(),
as: 'Avatar',
2018-02-16 04:19:54 -06:00
required: false
2017-12-14 10:38:41 -06:00
}
]
2017-12-14 03:07:57 -06:00
}
]
}
]
}
2019-04-23 02:50:57 -05:00
]
2017-12-14 03:07:57 -06:00
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_TAGS]: {
2019-04-23 02:50:57 -05:00
include: [ TagModel ]
2017-12-14 03:07:57 -06:00
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_BLACKLISTED]: {
2018-08-14 02:08:47 -05:00
include: [
{
2019-08-15 04:53:26 -05:00
attributes: [ 'id', 'reason', 'unfederated' ],
2019-04-23 02:50:57 -05:00
model: VideoBlacklistModel,
2018-08-14 02:08:47 -05:00
required: false
}
]
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_WEBTORRENT_FILES]: (withRedundancies = false) => {
2019-01-29 01:37:25 -06:00
let subInclude: any[] = []
if (withRedundancies === true) {
subInclude = [
{
attributes: [ 'fileUrl' ],
model: VideoRedundancyModel.unscoped(),
required: false
}
]
}
return {
include: [
{
model: VideoFileModel,
2021-02-25 04:17:53 -06:00
separate: true,
2019-01-29 01:37:25 -06:00
required: false,
include: subInclude
}
]
}
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => {
const subInclude: IncludeOptions[] = [
{
model: VideoFileModel,
required: false
}
]
2019-01-29 01:37:25 -06:00
if (withRedundancies === true) {
subInclude.push({
attributes: [ 'fileUrl' ],
model: VideoRedundancyModel.unscoped(),
required: false
})
2019-01-29 01:37:25 -06:00
}
return {
include: [
{
model: VideoStreamingPlaylistModel.unscoped(),
required: false,
2021-02-25 04:17:53 -06:00
separate: true,
2019-01-29 01:37:25 -06:00
include: subInclude
}
]
}
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_SCHEDULED_UPDATE]: {
include: [
{
2019-04-23 02:50:57 -05:00
model: ScheduleVideoUpdateModel.unscoped(),
required: false
}
]
2018-10-05 04:15:06 -05:00
},
2020-01-31 09:56:52 -06:00
[ScopeNames.WITH_USER_HISTORY]: (userId: number) => {
2018-10-05 04:15:06 -05:00
return {
include: [
{
attributes: [ 'currentTime' ],
model: UserVideoHistoryModel.unscoped(),
required: false,
where: {
userId
}
}
]
}
2017-12-14 03:07:57 -06:00
}
2019-04-23 02:50:57 -05:00
}))
2017-12-12 10:53:50 -06:00
@Table({
tableName: 'video',
2020-01-28 07:45:17 -06:00
indexes: [
buildTrigramSearchIndex('video_name_trigram', 'name'),
{ fields: [ 'createdAt' ] },
{
fields: [
{ name: 'publishedAt', order: 'DESC' },
{ name: 'id', order: 'ASC' }
]
},
{ fields: [ 'duration' ] },
2021-03-03 04:24:16 -06:00
{
fields: [
{ name: 'views', order: 'DESC' },
{ name: 'id', order: 'ASC' }
]
},
2020-01-28 07:45:17 -06:00
{ fields: [ 'channelId' ] },
{
fields: [ 'originallyPublishedAt' ],
where: {
originallyPublishedAt: {
[Op.ne]: null
}
}
},
{
fields: [ 'category' ], // We don't care videos with an unknown category
where: {
category: {
[Op.ne]: null
}
}
},
{
fields: [ 'licence' ], // We don't care videos with an unknown licence
where: {
licence: {
[Op.ne]: null
}
}
},
{
fields: [ 'language' ], // We don't care videos with an unknown language
where: {
language: {
[Op.ne]: null
}
}
},
{
fields: [ 'nsfw' ], // Most of the videos are not NSFW
where: {
nsfw: true
}
},
{
fields: [ 'remote' ], // Only index local videos
where: {
remote: false
}
},
{
fields: [ 'uuid' ],
unique: true
},
{
fields: [ 'url' ],
unique: true
}
]
2017-12-12 10:53:50 -06:00
})
2021-05-12 07:09:04 -05:00
export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
2017-12-12 10:53:50 -06:00
@AllowNull(false)
@Default(DataType.UUIDV4)
@IsUUID(4)
@Column(DataType.UUID)
uuid: string
@AllowNull(false)
@Is('VideoName', value => throwIfNotValid(value, isVideoNameValid, 'name'))
@Column
name: string
@AllowNull(true)
@Default(null)
@Column
category: number
@AllowNull(true)
@Default(null)
@Column
licence: number
@AllowNull(true)
@Default(null)
2018-04-23 07:39:52 -05:00
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.LANGUAGE.max))
language: string
2017-12-12 10:53:50 -06:00
@AllowNull(false)
@Is('VideoPrivacy', value => throwIfNotValid(value, isVideoPrivacyValid, 'privacy'))
@Column
2020-06-16 07:13:01 -05:00
privacy: VideoPrivacy
2017-12-12 10:53:50 -06:00
@AllowNull(false)
2018-01-03 03:12:36 -06:00
@Is('VideoNSFW', value => throwIfNotValid(value, isBooleanValid, 'NSFW boolean'))
2017-12-12 10:53:50 -06:00
@Column
nsfw: boolean
@AllowNull(true)
@Default(null)
2019-04-18 04:28:17 -05:00
@Is('VideoDescription', value => throwIfNotValid(value, isVideoDescriptionValid, 'description', true))
2017-12-12 10:53:50 -06:00
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max))
description: string
@AllowNull(true)
@Default(null)
2019-04-18 04:28:17 -05:00
@Is('VideoSupport', value => throwIfNotValid(value, isVideoSupportValid, 'support', true))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.SUPPORT.max))
support: string
2017-12-12 10:53:50 -06:00
@AllowNull(false)
@Is('VideoDuration', value => throwIfNotValid(value, isVideoDurationValid, 'duration'))
@Column
duration: number
@AllowNull(false)
@Default(0)
@IsInt
@Min(0)
@Column
views: number
@AllowNull(false)
@Default(0)
@IsInt
@Min(0)
@Column
likes: number
@AllowNull(false)
@Default(0)
@IsInt
@Min(0)
@Column
dislikes: number
@AllowNull(false)
@Column
remote: boolean
@AllowNull(false)
@Default(false)
@Column
isLive: boolean
@AllowNull(false)
2017-12-12 10:53:50 -06:00
@Is('VideoUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
url: string
2018-01-03 03:12:36 -06:00
@AllowNull(false)
@Column
commentsEnabled: boolean
@AllowNull(false)
@Column
downloadEnabled: boolean
@AllowNull(false)
@Column
waitTranscoding: boolean
@AllowNull(false)
@Default(null)
@Is('VideoState', value => throwIfNotValid(value, isVideoStateValid, 'state'))
@Column
state: VideoState
2017-12-12 10:53:50 -06:00
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@AllowNull(false)
2019-04-18 04:28:17 -05:00
@Default(DataType.NOW)
@Column
publishedAt: Date
2019-02-11 07:41:55 -06:00
@AllowNull(true)
@Default(null)
2019-01-12 07:41:45 -06:00
@Column
originallyPublishedAt: Date
2017-12-12 10:53:50 -06:00
@ForeignKey(() => VideoChannelModel)
@Column
channelId: number
@BelongsTo(() => VideoChannelModel, {
2016-12-11 14:50:51 -06:00
foreignKey: {
2017-12-14 10:38:41 -06:00
allowNull: true
2016-12-11 14:50:51 -06:00
},
onDelete: 'cascade'
2016-12-11 14:50:51 -06:00
})
2017-12-12 10:53:50 -06:00
VideoChannel: VideoChannelModel
2016-12-24 09:59:17 -06:00
2017-12-12 10:53:50 -06:00
@BelongsToMany(() => TagModel, {
2016-12-24 09:59:17 -06:00
foreignKey: 'videoId',
2017-12-12 10:53:50 -06:00
through: () => VideoTagModel,
onDelete: 'CASCADE'
2016-12-24 09:59:17 -06:00
})
2017-12-12 10:53:50 -06:00
Tags: TagModel[]
2017-01-04 13:59:23 -06:00
2021-02-18 03:15:11 -06:00
@BelongsToMany(() => TrackerModel, {
foreignKey: 'videoId',
through: () => VideoTrackerModel,
onDelete: 'CASCADE'
})
Trackers: TrackerModel[]
@HasMany(() => ThumbnailModel, {
foreignKey: {
name: 'videoId',
allowNull: true
},
hooks: true,
onDelete: 'cascade'
})
Thumbnails: ThumbnailModel[]
2019-02-26 03:55:40 -06:00
@HasMany(() => VideoPlaylistElementModel, {
foreignKey: {
name: 'videoId',
2019-07-31 08:57:32 -05:00
allowNull: true
2019-02-26 03:55:40 -06:00
},
2019-07-31 08:57:32 -05:00
onDelete: 'set null'
2019-02-26 03:55:40 -06:00
})
VideoPlaylistElements: VideoPlaylistElementModel[]
2017-12-12 10:53:50 -06:00
@HasMany(() => VideoAbuseModel, {
2017-01-04 13:59:23 -06:00
foreignKey: {
name: 'videoId',
allowNull: true
2017-01-04 13:59:23 -06:00
},
onDelete: 'set null'
2017-01-04 13:59:23 -06:00
})
2017-12-12 10:53:50 -06:00
VideoAbuses: VideoAbuseModel[]
2017-12-12 10:53:50 -06:00
@HasMany(() => VideoFileModel, {
foreignKey: {
name: 'videoId',
allowNull: true
},
2018-09-11 09:27:07 -05:00
hooks: true,
onDelete: 'cascade'
})
2017-12-12 10:53:50 -06:00
VideoFiles: VideoFileModel[]
2017-11-21 11:23:10 -06:00
2019-01-29 01:37:25 -06:00
@HasMany(() => VideoStreamingPlaylistModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
hooks: true,
onDelete: 'cascade'
})
VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
2017-12-12 10:53:50 -06:00
@HasMany(() => VideoShareModel, {
2017-11-21 11:23:10 -06:00
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
2017-12-12 10:53:50 -06:00
VideoShares: VideoShareModel[]
2017-11-23 09:55:13 -06:00
2017-12-12 10:53:50 -06:00
@HasMany(() => AccountVideoRateModel, {
2017-11-23 09:55:13 -06:00
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
2017-12-12 10:53:50 -06:00
AccountVideoRates: AccountVideoRateModel[]
2016-11-11 08:20:03 -06:00
2017-12-28 04:16:08 -06:00
@HasMany(() => VideoCommentModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade',
hooks: true
2017-12-28 04:16:08 -06:00
})
VideoComments: VideoCommentModel[]
2018-08-31 10:18:13 -05:00
@HasMany(() => VideoViewModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
2018-10-05 04:15:06 -05:00
onDelete: 'cascade'
2018-08-31 10:18:13 -05:00
})
VideoViews: VideoViewModel[]
2018-10-05 04:15:06 -05:00
@HasMany(() => UserVideoHistoryModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
UserVideoHistories: UserVideoHistoryModel[]
@HasOne(() => ScheduleVideoUpdateModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
ScheduleVideoUpdate: ScheduleVideoUpdateModel
2018-08-13 09:57:13 -05:00
@HasOne(() => VideoBlacklistModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
VideoBlacklist: VideoBlacklistModel
2020-10-27 10:06:24 -05:00
@HasOne(() => VideoLiveModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
VideoLive: VideoLiveModel
@HasOne(() => VideoImportModel, {
foreignKey: {
name: 'videoId',
allowNull: true
},
onDelete: 'set null'
})
VideoImport: VideoImportModel
2018-07-12 12:02:00 -05:00
@HasMany(() => VideoCaptionModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade',
hooks: true,
2020-01-31 09:56:52 -06:00
['separate' as any]: true
2018-07-12 12:02:00 -05:00
})
VideoCaptions: VideoCaptionModel[]
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
@HasOne(() => VideoJobInfoModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
VideoJobInfo: VideoJobInfoModel
@BeforeDestroy
2019-08-15 04:53:26 -05:00
static async sendDelete (instance: MVideoAccountLight, options) {
2021-03-09 09:10:52 -06:00
if (!instance.isOwned()) return undefined
2021-03-09 09:10:52 -06:00
// Lazy load channels
if (!instance.VideoChannel) {
instance.VideoChannel = await instance.$get('VideoChannel', {
include: [
ActorModel,
AccountModel
],
transaction: options.transaction
}) as MChannelAccountDefault
}
2021-03-09 09:10:52 -06:00
return sendDeleteVideo(instance, options.transaction)
}
2018-04-25 03:21:38 -05:00
@BeforeDestroy
2021-06-11 09:22:54 -05:00
static async removeFiles (instance: VideoModel, options) {
const tasks: Promise<any>[] = []
2016-11-11 08:20:03 -06:00
2018-07-30 10:02:40 -05:00
logger.info('Removing files of video %s.', instance.url)
2018-04-25 03:21:38 -05:00
2017-12-12 10:53:50 -06:00
if (instance.isOwned()) {
if (!Array.isArray(instance.VideoFiles)) {
2021-06-11 09:22:54 -05:00
instance.VideoFiles = await instance.$get('VideoFiles', { transaction: options.transaction })
}
2017-12-12 10:53:50 -06:00
// Remove physical files and torrents
instance.VideoFiles.forEach(file => {
2021-07-23 04:20:00 -05:00
tasks.push(instance.removeFileAndTorrent(file))
2017-12-12 10:53:50 -06:00
})
2019-01-29 01:37:25 -06:00
// Remove playlists file
2020-01-24 09:48:05 -06:00
if (!Array.isArray(instance.VideoStreamingPlaylists)) {
2021-06-11 09:22:54 -05:00
instance.VideoStreamingPlaylists = await instance.$get('VideoStreamingPlaylists', { transaction: options.transaction })
2020-01-24 09:48:05 -06:00
}
for (const p of instance.VideoStreamingPlaylists) {
tasks.push(instance.removeStreamingPlaylistFiles(p))
}
2017-12-12 10:53:50 -06:00
}
2018-04-25 03:21:38 -05:00
// Do not wait video deletion because we could be in a transaction
Promise.all(tasks)
2018-09-03 11:05:12 -05:00
.catch(err => {
logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err })
})
2018-04-25 03:21:38 -05:00
return undefined
2017-12-12 10:53:50 -06:00
}
2016-11-11 08:20:03 -06:00
@BeforeDestroy
static stopLiveIfNeeded (instance: VideoModel) {
if (!instance.isLive) return
2020-11-04 07:16:57 -06:00
logger.info('Stopping live of video %s after video deletion.', instance.uuid)
2021-06-11 09:22:54 -05:00
LiveManager.Instance.stopSessionOf(instance.id)
}
@BeforeDestroy
static invalidateCache (instance: VideoModel) {
ModelCache.Instance.invalidateCache('video', instance.id)
}
@BeforeDestroy
static async saveEssentialDataToAbuses (instance: VideoModel, options) {
const tasks: Promise<any>[] = []
if (!Array.isArray(instance.VideoAbuses)) {
2021-06-11 09:22:54 -05:00
instance.VideoAbuses = await instance.$get('VideoAbuses', { transaction: options.transaction })
if (instance.VideoAbuses.length === 0) return undefined
}
2020-07-07 03:57:04 -05:00
logger.info('Saving video abuses details of video %s.', instance.url)
2021-03-09 09:10:52 -06:00
if (!instance.Trackers) instance.Trackers = await instance.$get('Trackers', { transaction: options.transaction })
const details = instance.toFormattedDetailsJSON()
for (const abuse of instance.VideoAbuses) {
abuse.deletedVideo = details
tasks.push(abuse.save({ transaction: options.transaction }))
}
2021-06-11 09:22:54 -05:00
await Promise.all(tasks)
}
static listLocal (): Promise<MVideo[]> {
const query = {
where: {
remote: false
}
}
return VideoModel.findAll(query)
}
2017-12-14 10:38:41 -06:00
static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
2017-12-12 10:53:50 -06:00
function getRawQuery (select: string) {
const queryVideo = 'SELECT ' + select + ' FROM "video" AS "Video" ' +
'INNER JOIN "videoChannel" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' +
2017-12-14 10:38:41 -06:00
'INNER JOIN "account" AS "Account" ON "Account"."id" = "VideoChannel"."accountId" ' +
'WHERE "Account"."actorId" = ' + actorId
2017-12-12 10:53:50 -06:00
const queryVideoShare = 'SELECT ' + select + ' FROM "videoShare" AS "VideoShare" ' +
'INNER JOIN "video" AS "Video" ON "Video"."id" = "VideoShare"."videoId" ' +
2017-12-14 10:38:41 -06:00
'WHERE "VideoShare"."actorId" = ' + actorId
2017-12-12 10:53:50 -06:00
return `(${queryVideo}) UNION (${queryVideoShare})`
}
2017-12-12 10:53:50 -06:00
const rawQuery = getRawQuery('"Video"."id"')
const rawCountQuery = getRawQuery('COUNT("Video"."id") as "total"')
const query = {
distinct: true,
offset: start,
limit: count,
2021-10-22 09:39:37 -05:00
order: getVideoSort('-createdAt', [ 'Tags', 'name', 'ASC' ]),
2017-12-12 10:53:50 -06:00
where: {
id: {
2020-01-31 09:56:52 -06:00
[Op.in]: Sequelize.literal('(' + rawQuery + ')')
},
[Op.or]: getPrivaciesForFederation()
2017-12-12 10:53:50 -06:00
},
include: [
2018-07-12 12:02:00 -05:00
{
2021-04-08 08:04:14 -05:00
attributes: [ 'filename', 'language', 'fileUrl' ],
2018-07-12 12:02:00 -05:00
model: VideoCaptionModel.unscoped(),
required: false
},
2017-12-12 10:53:50 -06:00
{
attributes: [ 'id', 'url' ],
model: VideoShareModel.unscoped(),
2017-12-12 10:53:50 -06:00
required: false,
2018-05-28 05:13:00 -05:00
// We only want videos shared by this actor
where: {
2020-01-31 09:56:52 -06:00
[Op.and]: [
2018-05-28 05:13:00 -05:00
{
id: {
2020-01-31 09:56:52 -06:00
[Op.not]: null
2018-05-28 05:13:00 -05:00
}
},
{
actorId
}
]
},
2017-12-14 10:38:41 -06:00
include: [
{
attributes: [ 'id', 'url' ],
model: ActorModel.unscoped()
2017-12-14 10:38:41 -06:00
}
]
2017-12-12 10:53:50 -06:00
},
{
model: VideoChannelModel.unscoped(),
2017-12-12 10:53:50 -06:00
required: true,
include: [
{
attributes: [ 'name' ],
model: AccountModel.unscoped(),
required: true,
include: [
{
2018-05-28 05:13:00 -05:00
attributes: [ 'id', 'url', 'followersUrl' ],
model: ActorModel.unscoped(),
required: true
}
]
},
{
2018-05-28 05:13:00 -05:00
attributes: [ 'id', 'url', 'followersUrl' ],
model: ActorModel.unscoped(),
2017-12-12 10:53:50 -06:00
required: true
}
]
},
2020-11-02 08:43:44 -06:00
{
model: VideoStreamingPlaylistModel.unscoped(),
required: false,
include: [
{
model: VideoFileModel,
required: false
}
]
},
2020-11-06 04:22:12 -06:00
VideoLiveModel.unscoped(),
2017-12-12 10:53:50 -06:00
VideoFileModel,
TagModel
2017-12-12 10:53:50 -06:00
]
}
2017-12-12 10:53:50 -06:00
return Bluebird.all([
2019-04-23 02:50:57 -05:00
VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findAll(query),
VideoModel.sequelize.query<{ total: string }>(rawCountQuery, { type: QueryTypes.SELECT })
2017-12-12 10:53:50 -06:00
]).then(([ rows, totals ]) => {
// totals: totalVideos + totalVideoShares
let totalVideos = 0
let totalVideoShares = 0
2020-01-31 09:56:52 -06:00
if (totals[0]) totalVideos = parseInt(totals[0].total, 10)
if (totals[1]) totalVideoShares = parseInt(totals[1].total, 10)
2017-12-12 10:53:50 -06:00
const total = totalVideos + totalVideoShares
return {
data: rows,
total: total
}
})
}
2021-05-28 06:05:59 -05:00
static async listPublishedLiveUUIDs () {
2020-11-13 07:36:30 -06:00
const options = {
2021-05-28 06:05:59 -05:00
attributes: [ 'uuid' ],
2020-11-13 07:36:30 -06:00
where: {
isLive: true,
2021-05-26 05:03:44 -05:00
remote: false,
2020-11-13 07:36:30 -06:00
state: VideoState.PUBLISHED
}
}
2020-12-08 07:30:29 -06:00
const result = await VideoModel.findAll(options)
2021-05-28 06:05:59 -05:00
return result.map(v => v.uuid)
2020-11-13 07:36:30 -06:00
}
2021-01-20 08:28:34 -06:00
static listUserVideosForApi (options: {
accountId: number
start: number
count: number
sort: string
channelId?: number
isLive?: boolean
search?: string
2021-01-20 08:28:34 -06:00
}) {
const { accountId, channelId, start, count, sort, search, isLive } = options
2021-01-20 08:28:34 -06:00
2019-04-23 02:50:57 -05:00
function buildBaseQuery (): FindOptions {
const where: WhereOptions = {}
if (search) {
where.name = {
[Op.iLike]: '%' + search + '%'
}
}
2021-10-20 03:04:06 -05:00
if (exists(isLive)) {
where.isLive = isLive
}
const channelWhere = channelId
? { id: channelId }
: {}
const baseQuery = {
2019-04-23 02:50:57 -05:00
offset: start,
limit: count,
where,
2019-04-23 02:50:57 -05:00
order: getVideoSort(sort),
include: [
{
model: VideoChannelModel,
required: true,
where: channelWhere,
2019-04-23 02:50:57 -05:00
include: [
{
model: AccountModel,
where: {
id: accountId
},
required: true
}
]
}
]
}
return baseQuery
2017-12-12 10:53:50 -06:00
}
2017-10-16 03:05:49 -05:00
2019-04-23 02:50:57 -05:00
const countQuery = buildBaseQuery()
const findQuery = buildBaseQuery()
const findScopes: (string | ScopeOptions)[] = [
2019-04-26 03:20:58 -05:00
ScopeNames.WITH_SCHEDULED_UPDATE,
ScopeNames.WITH_BLACKLISTED,
ScopeNames.WITH_THUMBNAILS
]
2019-04-23 02:50:57 -05:00
return Promise.all([
VideoModel.count(countQuery),
2019-08-20 06:52:49 -05:00
VideoModel.scope(findScopes).findAll<MVideoForUser>(findQuery)
2019-04-23 02:50:57 -05:00
]).then(([ count, rows ]) => {
return {
2019-08-20 06:52:49 -05:00
data: rows,
2019-04-23 02:50:57 -05:00
total: count
}
})
2017-12-12 10:53:50 -06:00
}
2018-04-24 10:05:32 -05:00
static async listForApi (options: {
2020-01-31 09:56:52 -06:00
start: number
count: number
sort: string
2020-01-31 09:56:52 -06:00
nsfw: boolean
isLive?: boolean
isLocal?: boolean
include?: VideoInclude
hasFiles?: boolean // default false
2020-01-31 09:56:52 -06:00
categoryOneOf?: number[]
licenceOneOf?: number[]
languageOneOf?: string[]
tagsOneOf?: string[]
tagsAllOf?: string[]
2020-01-31 09:56:52 -06:00
accountId?: number
videoChannelId?: number
displayOnlyForFollower: DisplayOnlyForFollowerOptions | null
2020-01-31 09:56:52 -06:00
videoPlaylistId?: number
2020-01-31 09:56:52 -06:00
trendingDays?: number
2020-01-31 09:56:52 -06:00
user?: MUserAccountId
historyOfUser?: MUserId
2020-01-08 07:15:16 -06:00
countVideos?: boolean
search?: string
2020-01-08 07:15:16 -06:00
}) {
if (VideoModel.isPrivateInclude(options.include) && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) {
throw new Error('Try to filter all-local but no user has not the see all videos right')
}
2020-03-05 08:04:57 -06:00
const trendingDays = options.sort.endsWith('trending')
? CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS
: undefined
2021-07-29 04:54:38 -05:00
let trendingAlgorithm: string
if (options.sort.endsWith('hot')) trendingAlgorithm = 'hot'
if (options.sort.endsWith('best')) trendingAlgorithm = 'best'
const serverActor = await getServerActor()
const queryOptions = {
2021-07-29 07:17:03 -05:00
...pick(options, [
'start',
'count',
'sort',
'nsfw',
'isLive',
'categoryOneOf',
'licenceOneOf',
'languageOneOf',
'tagsOneOf',
'tagsAllOf',
'isLocal',
'include',
'displayOnlyForFollower',
'hasFiles',
2021-07-29 07:17:03 -05:00
'accountId',
'videoChannelId',
'videoPlaylistId',
'user',
'historyOfUser',
'search'
]),
serverAccountIdForBlock: serverActor.Account.id,
trendingDays,
2021-07-29 07:17:03 -05:00
trendingAlgorithm
2018-04-24 10:05:32 -05:00
}
2020-03-05 08:04:57 -06:00
return VideoModel.getAvailableForApi(queryOptions, options.countVideos)
}
2018-07-20 11:31:49 -05:00
static async searchAndPopulateAccountAndServer (options: {
2021-07-29 07:17:03 -05:00
start: number
count: number
sort: string
search?: string
host?: string
2018-07-20 11:31:49 -05:00
startDate?: string // ISO 8601
endDate?: string // ISO 8601
originallyPublishedStartDate?: string
originallyPublishedEndDate?: string
2018-07-20 11:31:49 -05:00
nsfw?: boolean
isLive?: boolean
isLocal?: boolean
include?: VideoInclude
2018-07-20 11:31:49 -05:00
categoryOneOf?: number[]
licenceOneOf?: number[]
languageOneOf?: string[]
tagsOneOf?: string[]
tagsAllOf?: string[]
durationMin?: number // seconds
durationMax?: number // seconds
2020-01-31 09:56:52 -06:00
user?: MUserAccountId
uuids?: string[]
displayOnlyForFollower: DisplayOnlyForFollowerOptions | null
2018-07-20 11:31:49 -05:00
}) {
const serverActor = await getServerActor()
const queryOptions = {
2021-07-29 07:17:03 -05:00
...pick(options, [
'include',
2021-07-29 07:17:03 -05:00
'nsfw',
'isLive',
'categoryOneOf',
'licenceOneOf',
'languageOneOf',
'tagsOneOf',
'tagsAllOf',
'user',
'isLocal',
2021-07-29 07:17:03 -05:00
'host',
'start',
'count',
'sort',
'startDate',
'endDate',
'originallyPublishedStartDate',
'originallyPublishedEndDate',
'durationMin',
'durationMax',
'uuids',
'search',
'displayOnlyForFollower'
2021-07-29 07:17:03 -05:00
]),
serverAccountIdForBlock: serverActor.Account.id
2018-04-24 10:05:32 -05:00
}
2020-03-05 08:04:57 -06:00
return VideoModel.getAvailableForApi(queryOptions)
}
2020-10-28 09:24:40 -05:00
static countLocalLives () {
const options = {
where: {
remote: false,
isLive: true,
state: {
[Op.ne]: VideoState.LIVE_ENDED
}
2020-10-28 09:24:40 -05:00
}
}
return VideoModel.count(options)
}
static countVideosUploadedByUserSince (userId: number, since: Date) {
const options = {
include: [
{
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
model: AccountModel.unscoped(),
required: true,
include: [
{
model: UserModel.unscoped(),
required: true,
where: {
id: userId
}
}
]
}
]
}
],
where: {
createdAt: {
[Op.gte]: since
}
}
}
return VideoModel.unscoped().count(options)
}
2020-10-28 09:24:40 -05:00
static countLivesOfAccount (accountId: number) {
const options = {
where: {
remote: false,
isLive: true,
state: {
[Op.ne]: VideoState.LIVE_ENDED
}
2020-10-28 09:24:40 -05:00
},
include: [
{
required: true,
model: VideoChannelModel.unscoped(),
where: {
accountId
}
}
]
}
return VideoModel.count(options)
}
2021-06-11 07:09:33 -05:00
static load (id: number | string, transaction?: Transaction): Promise<MVideoThumbnail> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2017-10-16 03:05:49 -05:00
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails' })
2017-12-12 10:53:50 -06:00
}
2017-10-16 03:05:49 -05:00
2021-06-11 07:09:33 -05:00
static loadWithBlacklist (id: number | string, transaction?: Transaction): Promise<MVideoThumbnailBlacklist> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2019-08-22 03:46:54 -05:00
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails-blacklist' })
2019-08-22 03:46:54 -05:00
}
2020-12-08 07:30:29 -06:00
static loadImmutableAttributes (id: number | string, t?: Transaction): Promise<MVideoImmutable> {
const fun = () => {
const query = {
where: buildWhereIdOrUUID(id),
transaction: t
}
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
}
return ModelCache.Instance.doCache({
cacheType: 'load-video-immutable-id',
key: '' + id,
deleteKey: 'video',
fun
})
}
2020-12-08 07:30:29 -06:00
static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Promise<MVideoImmutable> {
const fun = () => {
const query: FindOptions = {
where: {
url
},
transaction
}
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
}
return ModelCache.Instance.doCache({
cacheType: 'load-video-immutable-url',
key: url,
deleteKey: 'video',
fun
})
}
2021-06-11 07:09:33 -05:00
static loadOnlyId (id: number | string, transaction?: Transaction): Promise<MVideoId> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction, type: 'id' })
}
2021-06-11 07:09:33 -05:00
static loadWithFiles (id: number | string, transaction?: Transaction, logging?: boolean): Promise<MVideoWithAllFiles> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction, type: 'all-files', logging })
}
2017-10-31 05:52:52 -05:00
2021-06-11 07:09:33 -05:00
static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoThumbnail> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2019-01-29 01:37:25 -06:00
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ url, transaction, type: 'thumbnails' })
}
static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Promise<MVideoAccountLightBlacklistAllFiles> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ url, transaction, type: 'account-blacklist-files' })
}
static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Transaction, userId?: number): Promise<MVideoFullLight> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2019-01-29 01:37:25 -06:00
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction: t, type: 'full-light', userId })
2019-01-29 01:37:25 -06:00
}
2019-07-19 10:30:41 -05:00
static loadForGetAPI (parameters: {
2020-01-31 09:56:52 -06:00
id: number | string
transaction?: Transaction
2019-07-19 10:30:41 -05:00
userId?: number
2020-12-08 07:30:29 -06:00
}): Promise<MVideoDetails> {
const { id, transaction, userId } = parameters
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
2019-01-29 01:37:25 -06:00
2021-06-11 07:09:33 -05:00
return queryBuilder.queryVideo({ id, transaction, type: 'api', userId })
2017-12-28 04:16:08 -06:00
}
2018-02-28 11:04:46 -06:00
static async getStats () {
const totalLocalVideos = await VideoModel.count({
where: {
remote: false
}
})
let totalLocalVideoViews = await VideoModel.sum('views', {
where: {
remote: false
}
})
2020-03-13 07:43:26 -05:00
2018-02-28 11:04:46 -06:00
// Sequelize could return null...
if (!totalLocalVideoViews) totalLocalVideoViews = 0
const serverActor = await getServerActor()
2020-03-13 07:43:26 -05:00
const { total: totalVideos } = await VideoModel.listForApi({
start: 0,
count: 0,
sort: '-publishedAt',
nsfw: buildNSFWFilter(),
displayOnlyForFollower: {
actorId: serverActor.id,
orLocalVideos: true
}
2020-03-13 07:43:26 -05:00
})
2018-02-28 11:04:46 -06:00
return {
totalLocalVideos,
totalLocalVideoViews,
totalVideos
}
}
2018-08-29 09:26:25 -05:00
static incrementViews (id: number, views: number) {
return VideoModel.increment('views', {
by: views,
where: {
id
}
})
}
static updateRatesOf (videoId: number, type: VideoRateType, t: Transaction) {
const field = type === 'like'
? 'likes'
: 'dislikes'
const rawQuery = `UPDATE "video" SET "${field}" = ` +
'(' +
2021-06-11 07:45:58 -05:00
'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' +
') ' +
'WHERE "video"."id" = :videoId'
return AccountVideoRateModel.sequelize.query(rawQuery, {
transaction: t,
replacements: { videoId, rateType: type },
type: QueryTypes.UPDATE
})
}
static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) {
// Instances only share videos
const query = 'SELECT 1 FROM "videoShare" ' +
2020-01-31 09:56:52 -06:00
'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' +
2020-06-09 08:59:35 -05:00
'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' +
2020-01-31 09:56:52 -06:00
'LIMIT 1'
const options = {
2019-10-21 07:50:55 -05:00
type: QueryTypes.SELECT as QueryTypes.SELECT,
bind: { followerActorId, videoId },
raw: true
}
return VideoModel.sequelize.query(query, options)
.then(results => results.length === 1)
}
2021-06-11 07:45:58 -05:00
static bulkUpdateSupportField (ofChannel: MChannel, t: Transaction) {
const options = {
where: {
2021-06-11 07:45:58 -05:00
channelId: ofChannel.id
},
transaction: t
}
2021-06-11 07:45:58 -05:00
return VideoModel.update({ support: ofChannel.support }, options)
}
2020-12-08 07:30:29 -06:00
static getAllIdsFromChannel (videoChannel: MChannelId): Promise<number[]> {
const query = {
attributes: [ 'id' ],
where: {
channelId: videoChannel.id
}
}
return VideoModel.findAll(query)
2020-01-31 09:56:52 -06:00
.then(videos => videos.map(v => v.id))
}
2018-08-30 07:58:00 -05:00
// threshold corresponds to how many video the field should have to be returned
2018-09-14 04:52:23 -05:00
static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
const serverActor = await getServerActor()
2018-09-14 04:52:23 -05:00
2021-06-10 01:53:32 -05:00
const queryOptions: BuildVideosListQueryOptions = {
2020-03-05 08:04:57 -06:00
attributes: [ `"${field}"` ],
group: `GROUP BY "${field}"`,
having: `HAVING COUNT("${field}") >= ${threshold}`,
start: 0,
sort: 'random',
count,
serverAccountIdForBlock: serverActor.Account.id,
displayOnlyForFollower: {
actorId: serverActor.id,
orLocalVideos: true
}
2018-09-14 04:52:23 -05:00
}
2021-06-10 01:53:32 -05:00
const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize)
2018-08-30 07:58:00 -05:00
2021-06-10 01:53:32 -05:00
return queryBuilder.queryVideoIds(queryOptions)
.then(rows => rows.map(r => r[field]))
2018-08-30 07:58:00 -05:00
}
2018-09-14 02:57:21 -05:00
static buildTrendingQuery (trendingDays: number) {
return {
attributes: [],
subQuery: false,
model: VideoViewModel,
required: false,
where: {
startDate: {
2020-01-31 09:56:52 -06:00
[Op.gte]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
2018-09-14 02:57:21 -05:00
}
}
}
}
2018-10-05 04:15:06 -05:00
private static async getAvailableForApi (
2021-06-10 01:53:32 -05:00
options: BuildVideosListQueryOptions,
2018-10-05 04:15:06 -05:00
countVideos = true
2020-06-05 03:42:36 -05:00
): Promise<ResultList<VideoModel>> {
2020-03-09 08:44:44 -05:00
function getCount () {
if (countVideos !== true) return Promise.resolve(undefined)
2018-09-03 11:05:12 -05:00
2020-03-09 08:44:44 -05:00
const countOptions = Object.assign({}, options, { isCount: true })
2021-06-10 01:53:32 -05:00
const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize)
2021-06-10 01:53:32 -05:00
return queryBuilder.countVideoIds(countOptions)
2020-03-09 08:44:44 -05:00
}
function getModels () {
2020-03-13 07:43:26 -05:00
if (options.count === 0) return Promise.resolve([])
2021-06-10 01:53:32 -05:00
const queryBuilder = new VideosModelListQueryBuilder(VideoModel.sequelize)
2020-03-09 08:44:44 -05:00
2021-06-10 01:53:32 -05:00
return queryBuilder.queryVideos(options)
2020-03-09 08:44:44 -05:00
}
const [ count, rows ] = await Promise.all([ getCount(), getModels() ])
return {
data: rows,
total: count
}
}
private static isPrivateInclude (include: VideoInclude) {
return include & VideoInclude.BLACKLISTED ||
include & VideoInclude.BLOCKED_OWNER ||
include & VideoInclude.HIDDEN_PRIVACY ||
include & VideoInclude.NOT_PUBLISHED_STATE
}
2019-07-23 05:04:15 -05:00
isBlacklisted () {
return !!this.VideoBlacklist
}
2019-07-31 08:57:32 -05:00
isBlocked () {
2020-06-17 03:55:40 -05:00
return this.VideoChannel.Account.Actor.Server?.isBlocked() || this.VideoChannel.Account.isBlocked()
2019-07-31 08:57:32 -05:00
}
2020-01-31 09:56:52 -06:00
getQualityFileBy<T extends MVideoWithFile> (this: T, fun: (files: MVideoFile[], it: (file: MVideoFile) => number) => MVideoFile) {
2021-01-21 08:58:17 -06:00
// We first transcode to WebTorrent format, so try this array first
if (Array.isArray(this.VideoFiles) && this.VideoFiles.length !== 0) {
const file = fun(this.VideoFiles, file => file.resolution)
return Object.assign(file, { Video: this })
}
// No webtorrent files, try with streaming playlist files
if (Array.isArray(this.VideoStreamingPlaylists) && this.VideoStreamingPlaylists.length !== 0) {
const streamingPlaylistWithVideo = Object.assign(this.VideoStreamingPlaylists[0], { Video: this })
const file = fun(streamingPlaylistWithVideo.VideoFiles, file => file.resolution)
return Object.assign(file, { VideoStreamingPlaylist: streamingPlaylistWithVideo })
}
return undefined
2017-11-09 10:51:58 -06:00
}
2020-01-31 09:56:52 -06:00
getMaxQualityFile<T extends MVideoWithFile> (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo {
return this.getQualityFileBy(maxBy)
}
2020-01-31 09:56:52 -06:00
getMinQualityFile<T extends MVideoWithFile> (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo {
return this.getQualityFileBy(minBy)
}
2020-01-31 09:56:52 -06:00
getWebTorrentFile<T extends MVideoWithFile> (this: T, resolution: number): MVideoFileVideo {
2019-08-01 07:19:18 -05:00
if (Array.isArray(this.VideoFiles) === false) return undefined
const file = this.VideoFiles.find(f => f.resolution === resolution)
if (!file) return undefined
return Object.assign(file, { Video: this })
2019-08-01 07:19:18 -05:00
}
hasWebTorrentFiles () {
return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0
}
2021-06-08 10:29:45 -05:00
async addAndSaveThumbnail (thumbnail: MThumbnail, transaction?: Transaction) {
2019-04-23 02:50:57 -05:00
thumbnail.videoId = this.id
const savedThumbnail = await thumbnail.save({ transaction })
if (Array.isArray(this.Thumbnails) === false) this.Thumbnails = []
// Already have this thumbnail, skip
2019-04-23 02:50:57 -05:00
if (this.Thumbnails.find(t => t.id === savedThumbnail.id)) return
2019-04-23 02:50:57 -05:00
this.Thumbnails.push(savedThumbnail)
}
2019-04-23 02:50:57 -05:00
getMiniature () {
if (Array.isArray(this.Thumbnails) === false) return undefined
2019-04-23 02:50:57 -05:00
return this.Thumbnails.find(t => t.type === ThumbnailType.MINIATURE)
}
hasPreview () {
return !!this.getPreview()
}
getPreview () {
if (Array.isArray(this.Thumbnails) === false) return undefined
return this.Thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
2017-12-12 10:53:50 -06:00
}
2016-12-29 12:07:05 -06:00
2017-12-12 10:53:50 -06:00
isOwned () {
return this.remote === false
2017-10-30 04:16:27 -05:00
}
2018-12-26 03:36:24 -06:00
getWatchStaticPath () {
return buildVideoWatchPath({ shortUUID: uuidToShort(this.uuid) })
2018-12-26 03:36:24 -06:00
}
2018-07-12 12:02:00 -05:00
getEmbedStaticPath () {
2021-07-26 08:04:37 -05:00
return buildVideoEmbedPath(this)
2017-12-12 10:53:50 -06:00
}
2017-11-09 10:51:58 -06:00
2019-04-23 02:50:57 -05:00
getMiniatureStaticPath () {
const thumbnail = this.getMiniature()
if (!thumbnail) return null
return join(STATIC_PATHS.THUMBNAILS, thumbnail.filename)
2017-11-09 10:51:58 -06:00
}
2018-07-12 12:02:00 -05:00
getPreviewStaticPath () {
const preview = this.getPreview()
if (!preview) return null
// We use a local cache, so specify our cache endpoint instead of potential remote URL
2019-08-09 04:32:40 -05:00
return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename)
2017-12-12 10:53:50 -06:00
}
2019-08-21 07:31:57 -05:00
toFormattedJSON (this: MVideoFormattable, options?: VideoFormattingJSONOptions): Video {
return videoModelToFormattedJSON(this, options)
}
2019-08-21 07:31:57 -05:00
toFormattedDetailsJSON (this: MVideoFormattableDetails): VideoDetails {
return videoModelToFormattedDetailsJSON(this)
}
2021-02-18 04:22:35 -06:00
getFormattedVideoFilesJSON (includeMagnet = true): VideoFile[] {
2021-01-27 09:42:13 -06:00
let files: VideoFile[] = []
2020-08-24 09:11:37 -05:00
if (Array.isArray(this.VideoFiles)) {
2021-02-18 04:22:35 -06:00
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet)
2021-01-27 09:42:13 -06:00
files = files.concat(result)
2020-08-24 09:11:37 -05:00
}
for (const p of (this.VideoStreamingPlaylists || [])) {
2021-02-18 04:22:35 -06:00
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet)
2021-01-27 09:42:13 -06:00
files = files.concat(result)
2020-08-24 09:11:37 -05:00
}
2021-01-27 09:42:13 -06:00
return files
2017-12-12 10:53:50 -06:00
}
2017-11-09 10:51:58 -06:00
2020-09-17 06:59:02 -05:00
toActivityPubObject (this: MVideoAP): VideoObject {
return videoModelToActivityPubObject(this)
2017-12-12 10:53:50 -06:00
}
getTruncatedDescription () {
if (!this.description) return null
const maxLength = CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max
return peertubeTruncate(this.description, { length: maxLength })
}
getMaxQualityResolution () {
const file = this.getMaxQualityFile()
const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
2017-11-10 07:34:45 -06:00
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => {
return getVideoFileResolution(originalFilePath)
})
2017-12-12 10:53:50 -06:00
}
2017-11-10 07:34:45 -06:00
getDescriptionAPIPath () {
2017-12-12 10:53:50 -06:00
return `/api/${API_VERSION}/videos/${this.uuid}/description`
2016-12-11 14:50:51 -06:00
}
getHLSPlaylist (): MStreamingPlaylistFilesVideo {
2019-08-09 08:04:36 -05:00
if (!this.VideoStreamingPlaylists) return undefined
const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
playlist.Video = this
return playlist
2019-08-09 08:04:36 -05:00
}
setHLSPlaylist (playlist: MStreamingPlaylist) {
const toAdd = [ playlist ] as [ VideoStreamingPlaylistModel ]
2018-12-04 10:08:55 -06:00
if (Array.isArray(this.VideoStreamingPlaylists) === false || this.VideoStreamingPlaylists.length === 0) {
this.VideoStreamingPlaylists = toAdd
return
}
this.VideoStreamingPlaylists = this.VideoStreamingPlaylists
2020-01-31 09:56:52 -06:00
.filter(s => s.type !== VideoStreamingPlaylistType.HLS)
.concat(toAdd)
}
2021-07-23 04:20:00 -05:00
removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
const filePath = isRedundancy
? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile)
: VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile)
2021-07-23 04:20:00 -05:00
const promises: Promise<any>[] = [ remove(filePath) ]
if (!isRedundancy) promises.push(videoFile.removeTorrent())
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
promises.push(removeWebTorrentObjectStorage(videoFile))
}
2021-07-23 04:20:00 -05:00
return Promise.all(promises)
2016-12-11 14:50:51 -06:00
}
2020-01-24 09:48:05 -06:00
async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
const directoryPath = isRedundancy
? getHLSRedundancyDirectory(this)
: getHLSDirectory(this)
2019-01-29 01:37:25 -06:00
2020-01-24 09:48:05 -06:00
await remove(directoryPath)
if (isRedundancy !== true) {
2020-01-31 09:56:52 -06:00
const streamingPlaylistWithFiles = streamingPlaylist as MStreamingPlaylistFilesVideo
2020-01-24 09:48:05 -06:00
streamingPlaylistWithFiles.Video = this
if (!Array.isArray(streamingPlaylistWithFiles.VideoFiles)) {
streamingPlaylistWithFiles.VideoFiles = await streamingPlaylistWithFiles.$get('VideoFiles')
}
// Remove physical files and torrents
await Promise.all(
streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent())
2020-01-24 09:48:05 -06:00
)
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
await removeHLSObjectStorage(streamingPlaylist, this)
}
2020-01-24 09:48:05 -06:00
}
2019-01-29 01:37:25 -06:00
}
2018-08-22 09:15:35 -05:00
isOutdated () {
if (this.isOwned()) return false
2019-03-19 08:13:53 -05:00
return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL)
2018-08-22 09:15:35 -05:00
}
2019-12-12 08:47:47 -06:00
hasPrivacyForFederation () {
return isPrivacyForFederation(this.privacy)
2019-12-12 08:47:47 -06:00
}
2020-11-04 07:16:57 -06:00
hasStateForFederation () {
return isStateForFederation(this.state)
}
2019-12-12 08:47:47 -06:00
isNewVideo (newPrivacy: VideoPrivacy) {
return this.hasPrivacyForFederation() === false && isPrivacyForFederation(newPrivacy) === true
2019-12-12 08:47:47 -06:00
}
setAsRefreshed () {
return setAsUpdated('video', this.id)
}
2019-12-12 08:47:47 -06:00
requiresAuth () {
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
}
setPrivacy (newPrivacy: VideoPrivacy) {
if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
this.publishedAt = new Date()
}
this.privacy = newPrivacy
}
isConfidential () {
return this.privacy === VideoPrivacy.PRIVATE ||
this.privacy === VideoPrivacy.UNLISTED ||
this.privacy === VideoPrivacy.INTERNAL
}
async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
if (this.state === newState) throw new Error('Cannot use same state ' + newState)
this.state = newState
2016-12-24 09:59:17 -06:00
if (this.state === VideoState.PUBLISHED && isNewVideo) {
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
this.publishedAt = new Date()
}
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 01:26:20 -05:00
await this.save({ transaction })
}
2021-02-18 03:15:11 -06:00
getBandwidthBits (videoFile: MVideoFile) {
return Math.ceil((videoFile.size * 8) / this.duration)
2018-09-11 09:27:07 -05:00
}
2021-02-18 03:15:11 -06:00
getTrackerUrls () {
if (this.isOwned()) {
return [
WEBSERVER.URL + '/tracker/announce',
WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket'
]
}
2019-01-29 01:37:25 -06:00
2021-02-18 03:15:11 -06:00
return this.Trackers.map(t => t.url)
2019-01-29 01:37:25 -06:00
}
}