Improve `Video` AP compatibility

Compat with text/html descriptions
Compat with SPDX for licences
Compat with missing sensitive attribute
Compat with missing tag attribute
Compat with missing video file magnet URI
Compat with missing streaming playlist segmentsSha256Url
Compat with optional comments/likes/dislikes/shares URI in video object

Add more debug logs when the object is not valid
This commit is contained in:
Chocobozzz 2024-05-31 11:31:52 +02:00
parent 1e3a5b25c3
commit 7c9f07e140
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
13 changed files with 215 additions and 124 deletions

View File

@ -52,7 +52,7 @@ export class VideoDescriptionComponent implements OnChanges {
} }
private async setVideoDescriptionHTML () { private async setVideoDescriptionHTML () {
const html = await this.markdownService.textMarkdownToHTML({ markdown: this.video.description }) const html = await this.markdownService.textMarkdownToHTML({ markdown: this.video.description, withHtml: true, withEmoji: true })
this.videoHTMLDescription = this.markdownService.processVideoTimestamps(this.video.shortUUID, html) this.videoHTMLDescription = this.markdownService.processVideoTimestamps(this.video.shortUUID, html)
} }

View File

@ -129,7 +129,7 @@
<div *ngIf="hasMetadata()" [ngbNavOutlet]="navMetadata"></div> <div *ngIf="hasMetadata()" [ngbNavOutlet]="navMetadata"></div>
<div [hidden]="originalVideoFile" class="download-type"> <div [hidden]="originalVideoFile || !getVideoFile()?.torrentDownloadUrl" class="download-type">
<div class="peertube-radio-container"> <div class="peertube-radio-container">
<input type="radio" name="download" id="download-direct" [(ngModel)]="downloadType" value="direct"> <input type="radio" name="download" id="download-direct" [(ngModel)]="downloadType" value="direct">
<label i18n for="download-direct">Direct download</label> <label i18n for="download-direct">Direct download</label>

View File

@ -121,10 +121,16 @@ class P2pMediaLoaderPlugin extends Plugin {
logger.error(`Segment ${segment.id} error.`, err) logger.error(`Segment ${segment.id} error.`, err)
if (this.options.redundancyUrlManager) {
this.options.redundancyUrlManager.removeBySegmentUrl(segment.requestUrl) this.options.redundancyUrlManager.removeBySegmentUrl(segment.requestUrl)
}
}) })
this.statsP2PBytes.peersWithWebSeed = 1 + this.options.redundancyUrlManager.countBaseUrls() const redundancyUrlsCount = this.options.redundancyUrlManager
? this.options.redundancyUrlManager.countBaseUrls()
: 0
this.statsP2PBytes.peersWithWebSeed = 1 + redundancyUrlsCount
this.runStats() this.runStats()

View File

@ -1,14 +1,10 @@
import { Segment } from '@peertube/p2p-media-loader-core' import { Segment } from '@peertube/p2p-media-loader-core'
import { RedundancyUrlManager } from './redundancy-url-manager' import { RedundancyUrlManager } from './redundancy-url-manager'
function segmentUrlBuilderFactory (redundancyUrlManager: RedundancyUrlManager) { export function segmentUrlBuilderFactory (redundancyUrlManager: RedundancyUrlManager | null) {
return function segmentBuilder (segment: Segment) { return function segmentBuilder (segment: Segment) {
if (!redundancyUrlManager) return segment.url
return redundancyUrlManager.buildUrl(segment.url) return redundancyUrlManager.buildUrl(segment.url)
} }
} }
// ---------------------------------------------------------------------------
export {
segmentUrlBuilderFactory
}

View File

@ -1,14 +1,14 @@
import { HybridLoaderSettings } from '@peertube/p2p-media-loader-core' import { HybridLoaderSettings } from '@peertube/p2p-media-loader-core'
import { HlsJsEngineSettings } from '@peertube/p2p-media-loader-hlsjs' import { HlsJsEngineSettings } from '@peertube/p2p-media-loader-hlsjs'
import { logger } from '@root-helpers/logger'
import { LiveVideoLatencyMode } from '@peertube/peertube-models' import { LiveVideoLatencyMode } from '@peertube/peertube-models'
import { logger } from '@root-helpers/logger'
import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage'
import { getAverageBandwidthInStore } from '../../peertube-player-local-storage' import { getAverageBandwidthInStore } from '../../peertube-player-local-storage'
import { P2PMediaLoader, P2PMediaLoaderPluginOptions, PeerTubePlayerContructorOptions, PeerTubePlayerLoadOptions } from '../../types' import { P2PMediaLoader, P2PMediaLoaderPluginOptions, PeerTubePlayerContructorOptions, PeerTubePlayerLoadOptions } from '../../types'
import { getRtcConfig, isSameOrigin } from '../common' import { getRtcConfig, isSameOrigin } from '../common'
import { RedundancyUrlManager } from '../p2p-media-loader/redundancy-url-manager' import { RedundancyUrlManager } from '../p2p-media-loader/redundancy-url-manager'
import { segmentUrlBuilderFactory } from '../p2p-media-loader/segment-url-builder' import { segmentUrlBuilderFactory } from '../p2p-media-loader/segment-url-builder'
import { SegmentValidator } from '../p2p-media-loader/segment-validator' import { SegmentValidator } from '../p2p-media-loader/segment-validator'
import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage'
type ConstructorOptions = type ConstructorOptions =
Pick<PeerTubePlayerContructorOptions, 'pluginsManager' | 'serverUrl' | 'authorizationHeader'> & Pick<PeerTubePlayerContructorOptions, 'pluginsManager' | 'serverUrl' | 'authorizationHeader'> &
@ -25,15 +25,26 @@ export class HLSOptionsBuilder {
} }
async getPluginOptions () { async getPluginOptions () {
const redundancyUrlManager = new RedundancyUrlManager(this.options.hls.redundancyBaseUrls) const segmentsSha256Url = this.options.hls.segmentsSha256Url
const segmentValidator = new SegmentValidator({
segmentsSha256Url: this.options.hls.segmentsSha256Url, if (!segmentsSha256Url) {
logger.info('No segmentsSha256Url found. Disabling P2P & redundancy.')
}
const redundancyUrlManager = segmentsSha256Url
? new RedundancyUrlManager(this.options.hls.redundancyBaseUrls)
: null
const segmentValidator = segmentsSha256Url
? new SegmentValidator({
segmentsSha256Url,
authorizationHeader: this.options.authorizationHeader, authorizationHeader: this.options.authorizationHeader,
requiresUserAuth: this.options.requiresUserAuth, requiresUserAuth: this.options.requiresUserAuth,
serverUrl: this.options.serverUrl, serverUrl: this.options.serverUrl,
requiresPassword: this.options.requiresPassword, requiresPassword: this.options.requiresPassword,
videoPassword: this.options.videoPassword videoPassword: this.options.videoPassword
}) })
: null
const p2pMediaLoaderConfig = await this.options.pluginsManager.runHook( const p2pMediaLoaderConfig = await this.options.pluginsManager.runHook(
'filter:internal.player.p2p-media-loader.options.result', 'filter:internal.player.p2p-media-loader.options.result',
@ -45,7 +56,7 @@ export class HLSOptionsBuilder {
requiresUserAuth: this.options.requiresUserAuth, requiresUserAuth: this.options.requiresUserAuth,
videoFileToken: this.options.videoFileToken, videoFileToken: this.options.videoFileToken,
p2pEnabled: this.options.p2pEnabled, p2pEnabled: segmentsSha256Url && this.options.p2pEnabled,
redundancyUrlManager, redundancyUrlManager,
type: 'application/x-mpegURL', type: 'application/x-mpegURL',
@ -77,8 +88,8 @@ export class HLSOptionsBuilder {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
private getP2PMediaLoaderOptions (options: { private getP2PMediaLoaderOptions (options: {
redundancyUrlManager: RedundancyUrlManager redundancyUrlManager: RedundancyUrlManager | null
segmentValidator: SegmentValidator segmentValidator: SegmentValidator | null
}): HlsJsEngineSettings { }): HlsJsEngineSettings {
const { redundancyUrlManager, segmentValidator } = options const { redundancyUrlManager, segmentValidator } = options
@ -117,7 +128,9 @@ export class HLSOptionsBuilder {
else xhr.setRequestHeader('Authorization', this.options.authorizationHeader()) else xhr.setRequestHeader('Authorization', this.options.authorizationHeader())
}, },
segmentValidator: segmentValidator.validate.bind(segmentValidator), segmentValidator: segmentValidator
? segmentValidator.validate.bind(segmentValidator)
: null,
segmentUrlBuilder: segmentUrlBuilderFactory(redundancyUrlManager), segmentUrlBuilder: segmentUrlBuilderFactory(redundancyUrlManager),

View File

@ -198,14 +198,15 @@ type WebVideoPluginOptions = {
} }
type P2PMediaLoaderPluginOptions = { type P2PMediaLoaderPluginOptions = {
redundancyUrlManager: RedundancyUrlManager redundancyUrlManager: RedundancyUrlManager | null
segmentValidator: SegmentValidator | null
type: string type: string
src: string src: string
p2pEnabled: boolean p2pEnabled: boolean
loader: P2PMediaLoader loader: P2PMediaLoader
segmentValidator: SegmentValidator
requiresUserAuth: boolean requiresUserAuth: boolean
videoFileToken: () => string videoFileToken: () => string

View File

@ -8,10 +8,11 @@ import {
VideoState VideoState
} from '@peertube/peertube-models' } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js' import { logger } from '@server/helpers/logger.js'
import { spdxToPeertubeLicence } from '@server/helpers/video.js'
import validator from 'validator' import validator from 'validator'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
import { peertubeTruncate } from '../../core-utils.js' import { peertubeTruncate } from '../../core-utils.js'
import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js' import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js'
import { isLiveLatencyModeValid } from '../video-lives.js' import { isLiveLatencyModeValid } from '../video-lives.js'
import { import {
isVideoCommentsPolicyValid, isVideoCommentsPolicyValid,
@ -24,43 +25,28 @@ import {
} from '../videos.js' } from '../videos.js'
import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js' import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js'
function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { export function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
return isBaseActivityValid(activity, 'Update') && return isBaseActivityValid(activity, 'Update') &&
sanitizeAndCheckVideoTorrentObject(activity.object) sanitizeAndCheckVideoTorrentObject(activity.object)
} }
function sanitizeAndCheckVideoTorrentObject (video: VideoObject) { export function sanitizeAndCheckVideoTorrentObject (video: VideoObject) {
if (!video || video.type !== 'Video') return false if (!video || video.type !== 'Video') return false
if (!setValidRemoteTags(video)) { const fail = (field: string) => {
logger.debug('Video has invalid tags', { video }) logger.debug(`Video field is not valid to PeerTube: ${field}`, { video })
return false
}
if (!setValidRemoteVideoUrls(video)) {
logger.debug('Video has invalid urls', { video })
return false
}
if (!setRemoteVideoContent(video)) {
logger.debug('Video has invalid content', { video })
return false
}
if (!setValidAttributedTo(video)) {
logger.debug('Video has invalid attributedTo', { video })
return false
}
if (!setValidRemoteCaptions(video)) {
logger.debug('Video has invalid captions', { video })
return false
}
if (!setValidRemoteIcon(video)) {
logger.debug('Video has invalid icons', { video })
return false
}
if (!setValidStoryboard(video)) {
logger.debug('Video has invalid preview (storyboard)', { video })
return false return false
} }
if (!setValidRemoteTags(video)) return fail('tags')
if (!setValidRemoteVideoUrls(video)) return fail('urls')
if (!setRemoteVideoContent(video)) return fail('content')
if (!setValidAttributedTo(video)) return fail('attributedTo')
if (!setValidRemoteCaptions(video)) return fail('captions')
if (!setValidRemoteIcon(video)) return fail('icons')
if (!setValidStoryboard(video)) return fail('preview (storyboard)')
if (!setValidLicence(video)) return fail('licence')
// TODO: compat with < 6.1, remove in 7.0 // TODO: compat with < 6.1, remove in 7.0
if (!video.uuid && video['identifier']) video.uuid = video['identifier'] if (!video.uuid && video['identifier']) video.uuid = video['identifier']
@ -71,6 +57,7 @@ function sanitizeAndCheckVideoTorrentObject (video: VideoObject) {
if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false
if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false
if (!isBooleanValid(video.permanentLive)) video.permanentLive = false if (!isBooleanValid(video.permanentLive)) video.permanentLive = false
if (!isBooleanValid(video.sensitive)) video.sensitive = false
if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT
if (video.commentsPolicy) { if (video.commentsPolicy) {
@ -83,25 +70,31 @@ function sanitizeAndCheckVideoTorrentObject (video: VideoObject) {
video.commentsPolicy = VideoCommentPolicy.DISABLED video.commentsPolicy = VideoCommentPolicy.DISABLED
} }
return isActivityPubUrlValid(video.id) && if (!isActivityPubUrlValid(video.id)) return fail('id')
isVideoNameValid(video.name) && if (!isVideoNameValid(video.name)) return fail('name')
isActivityPubVideoDurationValid(video.duration) &&
isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && if (!isActivityPubVideoDurationValid(video.duration)) return fail('duration format')
isUUIDValid(video.uuid) && if (!isVideoDurationValid(video.duration.replace(/[^0-9]+/g, ''))) return fail('duration')
(!video.category || isRemoteNumberIdentifierValid(video.category)) &&
(!video.licence || isRemoteNumberIdentifierValid(video.licence)) && if (!isUUIDValid(video.uuid)) return fail('uuid')
(!video.language || isRemoteStringIdentifierValid(video.language)) &&
isVideoViewsValid(video.views) && if (exists(video.category) && !isRemoteNumberIdentifierValid(video.category)) return fail('category')
isBooleanValid(video.sensitive) && if (exists(video.language) && !isRemoteStringIdentifierValid(video.language)) return fail('language')
isDateValid(video.published) &&
isDateValid(video.updated) && if (!isVideoViewsValid(video.views)) return fail('views')
(!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && if (!isDateValid(video.published)) return fail('published')
(!video.uploadDate || isDateValid(video.uploadDate)) && if (!isDateValid(video.updated)) return fail('updated')
(!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
video.attributedTo.length !== 0 if (exists(video.originallyPublishedAt) && !isDateValid(video.originallyPublishedAt)) return fail('originallyPublishedAt')
if (exists(video.uploadDate) && !isDateValid(video.uploadDate)) return fail('uploadDate')
if (exists(video.content) && !isRemoteVideoContentValid(video.mediaType, video.content)) return fail('mediaType/content')
if (video.attributedTo.length === 0) return fail('attributedTo')
return true
} }
function isRemoteVideoUrlValid (url: any) { export function isRemoteVideoUrlValid (url: any) {
return url.type === 'Link' && return url.type === 'Link' &&
// Video file link // Video file link
( (
@ -133,44 +126,32 @@ function isRemoteVideoUrlValid (url: any) {
isAPVideoFileUrlMetadataObject(url) isAPVideoFileUrlMetadataObject(url)
} }
function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { export function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject {
return url && return url &&
url.type === 'Link' && url.type === 'Link' &&
url.mediaType === 'application/json' && url.mediaType === 'application/json' &&
isArray(url.rel) && url.rel.includes('metadata') isArray(url.rel) && url.rel.includes('metadata')
} }
function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { export function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject {
return isArray(url.rel) && return isArray(url.rel) &&
url.rel.includes('tracker') && url.rel.includes('tracker') &&
isActivityPubUrlValid(url.href) isActivityPubUrlValid(url.href)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Private
export {
isAPVideoFileUrlMetadataObject,
isAPVideoTrackerUrlObject,
isRemoteStringIdentifierValid,
isRemoteVideoUrlValid,
sanitizeAndCheckVideoTorrentObject,
sanitizeAndCheckVideoTorrentUpdateActivity
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function setValidRemoteTags (video: any) { function setValidRemoteTags (video: VideoObject) {
if (Array.isArray(video.tag) === false) return false if (Array.isArray(video.tag) === false) video.tag = []
video.tag = video.tag.filter(t => { video.tag = video.tag.filter(t => t.type === 'Hashtag' && isVideoTagValid(t.name))
return t.type === 'Hashtag' &&
isVideoTagValid(t.name)
})
return true return true
} }
function setValidRemoteCaptions (video: any) { function setValidRemoteCaptions (video: VideoObject) {
if (!video.subtitleLanguage) video.subtitleLanguage = [] if (!video.subtitleLanguage) video.subtitleLanguage = []
if (Array.isArray(video.subtitleLanguage) === false) return false if (Array.isArray(video.subtitleLanguage) === false) return false
@ -193,7 +174,7 @@ function isRemoteStringIdentifierValid (data: any) {
} }
function isRemoteVideoContentValid (mediaType: string, content: string) { function isRemoteVideoContentValid (mediaType: string, content: string) {
return mediaType === 'text/markdown' && isVideoDescriptionValid(content) return (mediaType === 'text/markdown' || mediaType === 'text/html') && isVideoDescriptionValid(content)
} }
function setValidRemoteIcon (video: any) { function setValidRemoteIcon (video: any) {
@ -219,7 +200,7 @@ function setValidRemoteVideoUrls (video: any) {
return true return true
} }
function setRemoteVideoContent (video: any) { function setRemoteVideoContent (video: VideoObject) {
if (video.content) { if (video.content) {
video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
} }
@ -227,6 +208,19 @@ function setRemoteVideoContent (video: any) {
return true return true
} }
function setValidLicence (video: VideoObject) {
if (!exists(video.licence)) return true
if (validator.default.isInt(video.licence.identifier)) return isRemoteNumberIdentifierValid(video.licence)
const spdx = spdxToPeertubeLicence(video.licence.identifier)
video.licence.identifier = spdx
? spdx + ''
: undefined
return true
}
function setValidStoryboard (video: VideoObject) { function setValidStoryboard (video: VideoObject) {
if (!video.preview) return true if (!video.preview) return true
if (!Array.isArray(video.preview)) return false if (!Array.isArray(video.preview)) return false

View File

@ -26,3 +26,27 @@ export function getExtFromMimetype (mimeTypes: { [id: string]: string | string[]
return value return value
} }
export function peertubeLicenceToSPDX (licence: number) {
return {
1: 'CC-BY-4.0',
2: 'CC-BY-SA-4.0',
3: 'CC-BY-ND-4.0',
4: 'CC-BY-NC-4.0',
5: 'CC-BY-NC-SA-4.0',
6: 'CC-BY-NC-ND-4.0',
7: 'CC0'
}[licence]
}
export function spdxToPeertubeLicence (licence: string) {
return {
'CC-BY-4.0': 1,
'CC-BY-SA-4.0': 2,
'CC-BY-ND-4.0': 3,
'CC-BY-NC-4.0': 4,
'CC-BY-NC-SA-4.0': 5,
'CC-BY-NC-ND-4.0': 6,
'CC0': 7
}[licence]
}

View File

@ -47,7 +47,7 @@ import { cpus } from 'os'
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 845 const LAST_MIGRATION_VERSION = 850
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -0,0 +1,25 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const { transaction } = utils
{
await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'segmentsSha256Filename', {
type: Sequelize.STRING,
defaultValue: null,
allowNull: true
}, { transaction })
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
down, up
}

View File

@ -16,7 +16,6 @@ import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validator
import { isArray } from '@server/helpers/custom-validators/misc.js' import { isArray } from '@server/helpers/custom-validators/misc.js'
import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos.js' import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos.js'
import { generateImageFilename } from '@server/helpers/image-utils.js' import { generateImageFilename } from '@server/helpers/image-utils.js'
import { logger } from '@server/helpers/logger.js'
import { getExtFromMimetype } from '@server/helpers/video.js' import { getExtFromMimetype } from '@server/helpers/video.js'
import { MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants.js' import { MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants.js'
import { generateTorrentFileName } from '@server/lib/paths.js' import { generateTorrentFileName } from '@server/lib/paths.js'
@ -58,21 +57,6 @@ export function getFileAttributesFromUrl (
const attributes: FilteredModelAttributes<VideoFileModel>[] = [] const attributes: FilteredModelAttributes<VideoFileModel>[] = []
for (const fileUrl of fileUrls) { for (const fileUrl of fileUrls) {
// Fetch associated magnet uri
const magnet = urls.filter(isAPMagnetUrlObject)
.find(u => u.height === fileUrl.height)
if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
const parsed = magnetUriDecode(magnet.href)
if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
throw new Error('Cannot parse magnet URI ' + magnet.href)
}
const torrentUrl = Array.isArray(parsed.xs)
? parsed.xs[0]
: parsed.xs
// Fetch associated metadata url, if any // Fetch associated metadata url, if any
const metadata = urls.filter(isAPVideoFileUrlMetadataObject) const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
.find(u => { .find(u => {
@ -84,14 +68,20 @@ export function getFileAttributesFromUrl (
const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
const resolution = fileUrl.height const resolution = fileUrl.height
const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id
const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null
const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist)
? videoOrPlaylist.id
: null
const { torrentFilename, infoHash, torrentUrl } = getTorrentRelatedInfo({ videoOrPlaylist, urls, fileUrl })
const attribute = { const attribute = {
extname, extname,
infoHash: parsed.infoHash,
resolution, resolution,
size: fileUrl.size, size: fileUrl.size,
fps: fileUrl.fps || -1, fps: fileUrl.fps || -1,
metadataUrl: metadata?.href, metadataUrl: metadata?.href,
width: fileUrl.width, width: fileUrl.width,
@ -101,9 +91,9 @@ export function getFileAttributesFromUrl (
filename: basename(fileUrl.href), filename: basename(fileUrl.href),
fileUrl: fileUrl.href, fileUrl: fileUrl.href,
infoHash,
torrentFilename,
torrentUrl, torrentUrl,
// Use our own torrent name since we proxify torrent requests
torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
// This is a video file owned by a video or by a streaming playlist // This is a video file owned by a video or by a streaming playlist
videoId, videoId,
@ -126,19 +116,17 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId, video
const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
if (!segmentsSha256UrlObject) {
logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
continue
}
const attribute = { const attribute = {
type: VideoStreamingPlaylistType.HLS, type: VideoStreamingPlaylistType.HLS,
playlistFilename: basename(playlistUrlObject.href), playlistFilename: basename(playlistUrlObject.href),
playlistUrl: playlistUrlObject.href, playlistUrl: playlistUrlObject.href,
segmentsSha256Filename: basename(segmentsSha256UrlObject.href), segmentsSha256Filename: segmentsSha256UrlObject
segmentsSha256Url: segmentsSha256UrlObject.href, ? basename(segmentsSha256UrlObject.href)
: null,
segmentsSha256Url: segmentsSha256UrlObject?.href ?? null,
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
@ -270,3 +258,41 @@ function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
function isAPHashTagObject (url: any): url is ActivityHashTagObject { function isAPHashTagObject (url: any): url is ActivityHashTagObject {
return url && url.type === 'Hashtag' return url && url.type === 'Hashtag'
} }
function getTorrentRelatedInfo (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
urls: (ActivityTagObject | ActivityUrlObject)[]
fileUrl: ActivityVideoUrlObject
}) {
const { urls, fileUrl, videoOrPlaylist } = options
// Fetch associated magnet uri
const magnet = urls.filter(isAPMagnetUrlObject)
.find(u => u.height === fileUrl.height)
if (!magnet) {
return {
torrentUrl: null,
torrentFilename: null,
infoHash: null
}
}
const magnetParsed = magnetUriDecode(magnet.href)
if (magnetParsed && isVideoFileInfoHashValid(magnetParsed.infoHash) === false) {
throw new Error('Info hash is not valid in magnet URI ' + magnet.href)
}
const torrentUrl = Array.isArray(magnetParsed.xs)
? magnetParsed.xs[0]
: magnetParsed.xs
return {
torrentUrl,
// Use our own torrent name since we proxify torrent requests
torrentFilename: generateTorrentFileName(videoOrPlaylist, fileUrl.height),
infoHash: magnetParsed.infoHash
}
}

View File

@ -56,6 +56,8 @@ async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVi
? fetchedVideo.likes ? fetchedVideo.likes
: fetchedVideo.dislikes : fetchedVideo.dislikes
if (!uri) return
logger.info('Sync %s of video %s', type, video.url) logger.info('Sync %s of video %s', type, video.url)
const { body } = await fetchAP<ActivityPubOrderedCollection<any>>(uri) const { body } = await fetchAP<ActivityPubOrderedCollection<any>>(uri)
@ -70,6 +72,7 @@ async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVi
function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
const uri = fetchedVideo.shares const uri = fetchedVideo.shares
if (!uri) return
if (!isSync) { if (!isSync) {
return createJob({ uri, videoId: video.id, type: 'video-shares' }) return createJob({ uri, videoId: video.id, type: 'video-shares' })
@ -84,6 +87,7 @@ function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean)
function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
const uri = fetchedVideo.comments const uri = fetchedVideo.comments
if (!uri) return
if (!isSync) { if (!isSync) {
return createJob({ uri, videoId: video.id, type: 'video-comments' }) return createJob({ uri, videoId: video.id, type: 'video-comments' })

View File

@ -84,7 +84,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
@Column @Column
p2pMediaLoaderPeerVersion: number p2pMediaLoaderPeerVersion: number
@AllowNull(false) @AllowNull(true)
@Column @Column
segmentsSha256Filename: string segmentsSha256Filename: string
@ -270,6 +270,8 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
getSha256SegmentsUrl (video: MVideo) { getSha256SegmentsUrl (video: MVideo) {
if (video.isOwned()) { if (video.isOwned()) {
if (!this.segmentsSha256Filename) return null
if (this.storage === FileStorage.OBJECT_STORAGE) { if (this.storage === FileStorage.OBJECT_STORAGE) {
return this.getSha256SegmentsObjectStorageUrl(video) return this.getSha256SegmentsObjectStorageUrl(video)
} }