Continue activitypub

This commit is contained in:
Chocobozzz 2017-11-10 14:34:45 +01:00
parent e4f97babf7
commit 0d0e8dd090
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
27 changed files with 1039 additions and 1086 deletions

View File

@ -1,26 +1,15 @@
import * as express from 'express' import * as express from 'express'
import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, ActivityType, RootActivity } from '../../../shared'
import {
processCreateActivity,
processUpdateActivity,
processFlagActivity
} from '../../lib'
import {
Activity,
ActivityType,
RootActivity,
ActivityPubCollection,
ActivityPubOrderedCollection
} from '../../../shared'
import {
signatureValidator,
checkSignature,
asyncMiddleware
} from '../../middlewares'
import { logger } from '../../helpers' import { logger } from '../../helpers'
import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity'
import { processCreateActivity, processFlagActivity, processUpdateActivity } from '../../lib'
import { processAddActivity } from '../../lib/activitypub/process-add'
import { asyncMiddleware, checkSignature, signatureValidator } from '../../middlewares'
import { activityPubValidator } from '../../middlewares/validators/activitypub/activity'
const processActivity: { [ P in ActivityType ]: (activity: Activity) => Promise<any> } = { const processActivity: { [ P in ActivityType ]: (activity: Activity) => Promise<any> } = {
Create: processCreateActivity, Create: processCreateActivity,
Add: processAddActivity,
Update: processUpdateActivity, Update: processUpdateActivity,
Flag: processFlagActivity Flag: processFlagActivity
} }
@ -30,7 +19,7 @@ const inboxRouter = express.Router()
inboxRouter.post('/', inboxRouter.post('/',
signatureValidator, signatureValidator,
asyncMiddleware(checkSignature), asyncMiddleware(checkSignature),
// inboxValidator, activityPubValidator,
asyncMiddleware(inboxController) asyncMiddleware(inboxController)
) )
@ -54,6 +43,9 @@ async function inboxController (req: express.Request, res: express.Response, nex
activities = [ rootActivity as Activity ] activities = [ rootActivity as Activity ]
} }
// Only keep activities we are able to process
activities = activities.filter(a => isActivityValid(a))
await processActivities(activities) await processActivities(activities)
res.status(204).end() res.status(204).end()

View File

@ -1,69 +1,69 @@
import * as express from 'express' // import * as express from 'express'
//
import { database as db } from '../../../initializers/database' // import { database as db } from '../../../initializers/database'
import { // import {
checkSignature, // checkSignature,
signatureValidator, // signatureValidator,
setBodyHostPort, // setBodyHostPort,
remotePodsAddValidator, // remotePodsAddValidator,
asyncMiddleware // asyncMiddleware
} from '../../../middlewares' // } from '../../../middlewares'
import { sendOwnedDataToPod } from '../../../lib' // import { sendOwnedDataToPod } from '../../../lib'
import { getMyPublicCert, getFormattedObjects } from '../../../helpers' // import { getMyPublicCert, getFormattedObjects } from '../../../helpers'
import { CONFIG } from '../../../initializers' // import { CONFIG } from '../../../initializers'
import { PodInstance } from '../../../models' // import { PodInstance } from '../../../models'
import { PodSignature, Pod as FormattedPod } from '../../../../shared' // import { PodSignature, Pod as FormattedPod } from '../../../../shared'
//
const remotePodsRouter = express.Router() // const remotePodsRouter = express.Router()
//
remotePodsRouter.post('/remove', // remotePodsRouter.post('/remove',
signatureValidator, // signatureValidator,
checkSignature, // checkSignature,
asyncMiddleware(removePods) // asyncMiddleware(removePods)
) // )
//
remotePodsRouter.post('/list', // remotePodsRouter.post('/list',
asyncMiddleware(remotePodsList) // asyncMiddleware(remotePodsList)
) // )
//
remotePodsRouter.post('/add', // remotePodsRouter.post('/add',
setBodyHostPort, // We need to modify the host before running the validator! // setBodyHostPort, // We need to modify the host before running the validator!
remotePodsAddValidator, // remotePodsAddValidator,
asyncMiddleware(addPods) // asyncMiddleware(addPods)
) // )
//
// --------------------------------------------------------------------------- // // ---------------------------------------------------------------------------
//
export { // export {
remotePodsRouter // remotePodsRouter
} // }
//
// --------------------------------------------------------------------------- // // ---------------------------------------------------------------------------
//
async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) { // async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) {
const information = req.body // const information = req.body
//
const pod = db.Pod.build(information) // const pod = db.Pod.build(information)
const podCreated = await pod.save() // const podCreated = await pod.save()
//
await sendOwnedDataToPod(podCreated.id) // await sendOwnedDataToPod(podCreated.id)
//
const cert = await getMyPublicCert() // const cert = await getMyPublicCert()
return res.json({ cert, email: CONFIG.ADMIN.EMAIL }) // return res.json({ cert, email: CONFIG.ADMIN.EMAIL })
} // }
//
async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) { // async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) {
const pods = await db.Pod.list() // const pods = await db.Pod.list()
//
return res.json(getFormattedObjects<FormattedPod, PodInstance>(pods, pods.length)) // return res.json(getFormattedObjects<FormattedPod, PodInstance>(pods, pods.length))
} // }
//
async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) { // async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) {
const signature: PodSignature = req.body.signature // const signature: PodSignature = req.body.signature
const host = signature.host // const host = signature.host
//
const pod = await db.Pod.loadByHost(host) // const pod = await db.Pod.loadByHost(host)
await pod.destroy() // await pod.destroy()
//
return res.type('json').status(204).end() // return res.type('json').status(204).end()
} // }

View File

@ -1,589 +1,339 @@
import * as express from 'express' // import * as express from 'express'
import * as Bluebird from 'bluebird' // import * as Bluebird from 'bluebird'
import * as Sequelize from 'sequelize' // import * as Sequelize from 'sequelize'
//
import { database as db } from '../../../initializers/database' // import { database as db } from '../../../initializers/database'
import { // import {
REQUEST_ENDPOINT_ACTIONS, // REQUEST_ENDPOINT_ACTIONS,
REQUEST_ENDPOINTS, // REQUEST_ENDPOINTS,
REQUEST_VIDEO_EVENT_TYPES, // REQUEST_VIDEO_EVENT_TYPES,
REQUEST_VIDEO_QADU_TYPES // REQUEST_VIDEO_QADU_TYPES
} from '../../../initializers' // } from '../../../initializers'
import { // import {
checkSignature, // checkSignature,
signatureValidator, // signatureValidator,
remoteVideosValidator, // remoteVideosValidator,
remoteQaduVideosValidator, // remoteQaduVideosValidator,
remoteEventsVideosValidator // remoteEventsVideosValidator
} from '../../../middlewares' // } from '../../../middlewares'
import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers' // import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers'
import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib' // import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib'
import { PodInstance, VideoFileInstance } from '../../../models' // import { PodInstance, VideoFileInstance } from '../../../models'
import { // import {
RemoteVideoRequest, // RemoteVideoRequest,
RemoteVideoCreateData, // RemoteVideoCreateData,
RemoteVideoUpdateData, // RemoteVideoUpdateData,
RemoteVideoRemoveData, // RemoteVideoRemoveData,
RemoteVideoReportAbuseData, // RemoteVideoReportAbuseData,
RemoteQaduVideoRequest, // RemoteQaduVideoRequest,
RemoteQaduVideoData, // RemoteQaduVideoData,
RemoteVideoEventRequest, // RemoteVideoEventRequest,
RemoteVideoEventData, // RemoteVideoEventData,
RemoteVideoChannelCreateData, // RemoteVideoChannelCreateData,
RemoteVideoChannelUpdateData, // RemoteVideoChannelUpdateData,
RemoteVideoChannelRemoveData, // RemoteVideoChannelRemoveData,
RemoteVideoAuthorRemoveData, // RemoteVideoAuthorRemoveData,
RemoteVideoAuthorCreateData // RemoteVideoAuthorCreateData
} from '../../../../shared' // } from '../../../../shared'
import { VideoInstance } from '../../../models/video/video-interface' // import { VideoInstance } from '../../../models/video/video-interface'
//
const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] // const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
//
// Functions to call when processing a remote request // // Functions to call when processing a remote request
// FIXME: use RemoteVideoRequestType as id type // // FIXME: use RemoteVideoRequestType as id type
const functionsHash: { [ id: string ]: (...args) => Promise<any> } = {} // const functionsHash: { [ id: string ]: (...args) => Promise<any> } = {}
functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper // functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper
functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper // functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper
functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper // functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper
functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper // functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper
functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper // functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper
functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper // functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper
functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper // functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper
functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper // functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper
functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper // functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper
//
const remoteVideosRouter = express.Router() // const remoteVideosRouter = express.Router()
//
remoteVideosRouter.post('/', // remoteVideosRouter.post('/',
signatureValidator, // signatureValidator,
checkSignature, // checkSignature,
remoteVideosValidator, // remoteVideosValidator,
remoteVideos // remoteVideos
) // )
//
remoteVideosRouter.post('/qadu', // remoteVideosRouter.post('/qadu',
signatureValidator, // signatureValidator,
checkSignature, // checkSignature,
remoteQaduVideosValidator, // remoteQaduVideosValidator,
remoteVideosQadu // remoteVideosQadu
) // )
//
remoteVideosRouter.post('/events', // remoteVideosRouter.post('/events',
signatureValidator, // signatureValidator,
checkSignature, // checkSignature,
remoteEventsVideosValidator, // remoteEventsVideosValidator,
remoteVideosEvents // remoteVideosEvents
) // )
//
// --------------------------------------------------------------------------- // // ---------------------------------------------------------------------------
//
export { // export {
remoteVideosRouter // remoteVideosRouter
} // }
//
// --------------------------------------------------------------------------- // // ---------------------------------------------------------------------------
//
function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) { // function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
const requests: RemoteVideoRequest[] = req.body.data // const requests: RemoteVideoRequest[] = req.body.data
const fromPod = res.locals.secure.pod // const fromPod = res.locals.secure.pod
//
// We need to process in the same order to keep consistency // // We need to process in the same order to keep consistency
Bluebird.each(requests, request => { // Bluebird.each(requests, request => {
const data = request.data // const data = request.data
//
// Get the function we need to call in order to process the request // // Get the function we need to call in order to process the request
const fun = functionsHash[request.type] // const fun = functionsHash[request.type]
if (fun === undefined) { // if (fun === undefined) {
logger.error('Unknown remote request type %s.', request.type) // logger.error('Unknown remote request type %s.', request.type)
return // return
} // }
//
return fun.call(this, data, fromPod) // return fun.call(this, data, fromPod)
}) // })
.catch(err => logger.error('Error managing remote videos.', err)) // .catch(err => logger.error('Error managing remote videos.', err))
//
// Don't block the other pod // // Don't block the other pod
return res.type('json').status(204).end() // return res.type('json').status(204).end()
} // }
//
function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) { // function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) {
const requests: RemoteQaduVideoRequest[] = req.body.data // const requests: RemoteQaduVideoRequest[] = req.body.data
const fromPod = res.locals.secure.pod // const fromPod = res.locals.secure.pod
//
Bluebird.each(requests, request => { // Bluebird.each(requests, request => {
const videoData = request.data // const videoData = request.data
//
return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod) // return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod)
}) // })
.catch(err => logger.error('Error managing remote videos.', err)) // .catch(err => logger.error('Error managing remote videos.', err))
//
return res.type('json').status(204).end() // return res.type('json').status(204).end()
} // }
//
function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) { // function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) {
const requests: RemoteVideoEventRequest[] = req.body.data // const requests: RemoteVideoEventRequest[] = req.body.data
const fromPod = res.locals.secure.pod // const fromPod = res.locals.secure.pod
//
Bluebird.each(requests, request => { // Bluebird.each(requests, request => {
const eventData = request.data // const eventData = request.data
//
return processVideosEventsRetryWrapper(eventData, fromPod) // return processVideosEventsRetryWrapper(eventData, fromPod)
}) // })
.catch(err => logger.error('Error managing remote videos.', err)) // .catch(err => logger.error('Error managing remote videos.', err))
//
return res.type('json').status(204).end() // return res.type('json').status(204).end()
} // }
//
async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) { // async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) {
const options = { // const options = {
arguments: [ eventData, fromPod ], // arguments: [ eventData, fromPod ],
errorMessage: 'Cannot process videos events with many retries.' // errorMessage: 'Cannot process videos events with many retries.'
} // }
//
await retryTransactionWrapper(processVideosEvents, options) // await retryTransactionWrapper(processVideosEvents, options)
} // }
//
async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) { // async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) {
await db.sequelize.transaction(async t => { // await db.sequelize.transaction(async t => {
const sequelizeOptions = { transaction: t } // const sequelizeOptions = { transaction: t }
const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t) // const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t)
//
let columnToUpdate // let columnToUpdate
let qaduType // let qaduType
//
switch (eventData.eventType) { // switch (eventData.eventType) {
case REQUEST_VIDEO_EVENT_TYPES.VIEWS: // case REQUEST_VIDEO_EVENT_TYPES.VIEWS:
columnToUpdate = 'views' // columnToUpdate = 'views'
qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS // qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS
break // break
//
case REQUEST_VIDEO_EVENT_TYPES.LIKES: // case REQUEST_VIDEO_EVENT_TYPES.LIKES:
columnToUpdate = 'likes' // columnToUpdate = 'likes'
qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES // qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES
break // break
//
case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: // case REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
columnToUpdate = 'dislikes' // columnToUpdate = 'dislikes'
qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES // qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES
break // break
//
default: // default:
throw new Error('Unknown video event type.') // throw new Error('Unknown video event type.')
} // }
//
const query = {} // const query = {}
query[columnToUpdate] = eventData.count // query[columnToUpdate] = eventData.count
//
await videoInstance.increment(query, sequelizeOptions) // await videoInstance.increment(query, sequelizeOptions)
//
const qadusParams = [ // const qadusParams = [
{ // {
videoId: videoInstance.id, // videoId: videoInstance.id,
type: qaduType // type: qaduType
} // }
] // ]
await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) // await quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
}) // })
//
logger.info('Remote video event processed for video with uuid %s.', eventData.uuid) // logger.info('Remote video event processed for video with uuid %s.', eventData.uuid)
} // }
//
async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) { // async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
const options = { // const options = {
arguments: [ videoData, fromPod ], // arguments: [ videoData, fromPod ],
errorMessage: 'Cannot update quick and dirty the remote video with many retries.' // errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
} // }
//
await retryTransactionWrapper(quickAndDirtyUpdateVideo, options) // await retryTransactionWrapper(quickAndDirtyUpdateVideo, options)
} // }
//
async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { // async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
let videoUUID = '' // let videoUUID = ''
//
await db.sequelize.transaction(async t => { // await db.sequelize.transaction(async t => {
const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t) // const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t)
const sequelizeOptions = { transaction: t } // const sequelizeOptions = { transaction: t }
//
videoUUID = videoInstance.uuid // videoUUID = videoInstance.uuid
//
if (videoData.views) { // if (videoData.views) {
videoInstance.set('views', videoData.views) // videoInstance.set('views', videoData.views)
} // }
//
if (videoData.likes) { // if (videoData.likes) {
videoInstance.set('likes', videoData.likes) // videoInstance.set('likes', videoData.likes)
} // }
//
if (videoData.dislikes) { // if (videoData.dislikes) {
videoInstance.set('dislikes', videoData.dislikes) // videoInstance.set('dislikes', videoData.dislikes)
} // }
//
await videoInstance.save(sequelizeOptions) // await videoInstance.save(sequelizeOptions)
}) // })
//
logger.info('Remote video with uuid %s quick and dirty updated', videoUUID) // logger.info('Remote video with uuid %s quick and dirty updated', videoUUID)
} // }
//
// Handle retries on fail // async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { // const options = {
const options = { // arguments: [ videoToRemoveData, fromPod ],
arguments: [ videoToCreateData, fromPod ], // errorMessage: 'Cannot remove the remote video channel with many retries.'
errorMessage: 'Cannot insert the remote video with many retries.' // }
} //
// await retryTransactionWrapper(removeRemoteVideo, options)
await retryTransactionWrapper(addRemoteVideo, options) // }
} //
// async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { // logger.debug('Removing remote video "%s".', videoToRemoveData.uuid)
logger.debug('Adding remote video "%s".', videoToCreateData.uuid) //
// await db.sequelize.transaction(async t => {
await db.sequelize.transaction(async t => { // // We need the instance because we have to remove some other stuffs (thumbnail etc)
const sequelizeOptions = { // const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t)
transaction: t // await videoInstance.destroy({ transaction: t })
} // })
//
const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) // logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid)
if (videoFromDatabase) throw new Error('UUID already exists.') // }
//
const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) // async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') // const options = {
// arguments: [ authorAttributesToRemove, fromPod ],
const tags = videoToCreateData.tags // errorMessage: 'Cannot remove the remote video author with many retries.'
const tagInstances = await db.Tag.findOrCreateTags(tags, t) // }
//
const videoData = { // await retryTransactionWrapper(removeRemoteVideoAuthor, options)
name: videoToCreateData.name, // }
uuid: videoToCreateData.uuid, //
category: videoToCreateData.category, // async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
licence: videoToCreateData.licence, // logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid)
language: videoToCreateData.language, //
nsfw: videoToCreateData.nsfw, // await db.sequelize.transaction(async t => {
description: videoToCreateData.truncatedDescription, // const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t)
channelId: videoChannel.id, // await videoAuthor.destroy({ transaction: t })
duration: videoToCreateData.duration, // })
createdAt: videoToCreateData.createdAt, //
// FIXME: updatedAt does not seems to be considered by Sequelize // logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid)
updatedAt: videoToCreateData.updatedAt, // }
views: videoToCreateData.views, //
likes: videoToCreateData.likes, // async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
dislikes: videoToCreateData.dislikes, // const options = {
remote: true, // arguments: [ videoChannelAttributesToRemove, fromPod ],
privacy: videoToCreateData.privacy // errorMessage: 'Cannot remove the remote video channel with many retries.'
} // }
//
const video = db.Video.build(videoData) // await retryTransactionWrapper(removeRemoteVideoChannel, options)
await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) // }
const videoCreated = await video.save(sequelizeOptions) //
// async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
const tasks = [] // logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid)
for (const fileData of videoToCreateData.files) { //
const videoFileInstance = db.VideoFile.build({ // await db.sequelize.transaction(async t => {
extname: fileData.extname, // const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t)
infoHash: fileData.infoHash, // await videoChannel.destroy({ transaction: t })
resolution: fileData.resolution, // })
size: fileData.size, //
videoId: videoCreated.id // logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid)
}) // }
//
tasks.push(videoFileInstance.save(sequelizeOptions)) // async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
} // const options = {
// arguments: [ reportData, fromPod ],
await Promise.all(tasks) // errorMessage: 'Cannot create remote abuse video with many retries.'
// }
await videoCreated.setTags(tagInstances, sequelizeOptions) //
}) // await retryTransactionWrapper(reportAbuseRemoteVideo, options)
// }
logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) //
} // async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
// logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID)
// Handle retries on fail //
async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { // await db.sequelize.transaction(async t => {
const options = { // const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t)
arguments: [ videoAttributesToUpdate, fromPod ], // const videoAbuseData = {
errorMessage: 'Cannot update the remote video with many retries' // reporterUsername: reportData.reporterUsername,
} // reason: reportData.reportReason,
// reporterPodId: fromPod.id,
await retryTransactionWrapper(updateRemoteVideo, options) // videoId: videoInstance.id
} // }
//
async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { // await db.VideoAbuse.create(videoAbuseData)
logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) //
let videoInstance: VideoInstance // })
let videoFieldsSave: object //
// logger.info('Remote abuse for video uuid %s created', reportData.videoUUID)
try { // }
await db.sequelize.transaction(async t => { //
const sequelizeOptions = { // async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) {
transaction: t // try {
} // const video = await db.Video.loadLocalVideoByUUID(id, t)
//
const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t) // if (!video) throw new Error('Video ' + id + ' not found')
videoFieldsSave = videoInstance.toJSON() //
const tags = videoAttributesToUpdate.tags // return video
// } catch (err) {
const tagInstances = await db.Tag.findOrCreateTags(tags, t) // logger.error('Cannot load owned video from id.', { error: err.stack, id })
// throw err
videoInstance.set('name', videoAttributesToUpdate.name) // }
videoInstance.set('category', videoAttributesToUpdate.category) // }
videoInstance.set('licence', videoAttributesToUpdate.licence) //
videoInstance.set('language', videoAttributesToUpdate.language) // async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) {
videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) // try {
videoInstance.set('description', videoAttributesToUpdate.truncatedDescription) // const video = await db.Video.loadByHostAndUUID(podHost, uuid, t)
videoInstance.set('duration', videoAttributesToUpdate.duration) // if (!video) throw new Error('Video not found')
videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) //
videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) // return video
videoInstance.set('views', videoAttributesToUpdate.views) // } catch (err) {
videoInstance.set('likes', videoAttributesToUpdate.likes) // logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid })
videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) // throw err
videoInstance.set('privacy', videoAttributesToUpdate.privacy) // }
// }
await videoInstance.save(sequelizeOptions)
// Remove old video files
const videoFileDestroyTasks: Bluebird<void>[] = []
for (const videoFile of videoInstance.VideoFiles) {
videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
}
await Promise.all(videoFileDestroyTasks)
const videoFileCreateTasks: Bluebird<VideoFileInstance>[] = []
for (const fileData of videoAttributesToUpdate.files) {
const videoFileInstance = db.VideoFile.build({
extname: fileData.extname,
infoHash: fileData.infoHash,
resolution: fileData.resolution,
size: fileData.size,
videoId: videoInstance.id
})
videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions))
}
await Promise.all(videoFileCreateTasks)
await videoInstance.setTags(tagInstances, sequelizeOptions)
})
logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)
} catch (err) {
if (videoInstance !== undefined && videoFieldsSave !== undefined) {
resetSequelizeInstance(videoInstance, videoFieldsSave)
}
// This is just a debug because we will retry the insert
logger.debug('Cannot update the remote video.', err)
throw err
}
}
async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
const options = {
arguments: [ videoToRemoveData, fromPod ],
errorMessage: 'Cannot remove the remote video channel with many retries.'
}
await retryTransactionWrapper(removeRemoteVideo, options)
}
async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
logger.debug('Removing remote video "%s".', videoToRemoveData.uuid)
await db.sequelize.transaction(async t => {
// We need the instance because we have to remove some other stuffs (thumbnail etc)
const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t)
await videoInstance.destroy({ transaction: t })
})
logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid)
}
async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
const options = {
arguments: [ authorToCreateData, fromPod ],
errorMessage: 'Cannot insert the remote video author with many retries.'
}
await retryTransactionWrapper(addRemoteVideoAuthor, options)
}
async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
logger.debug('Adding remote video author "%s".', authorToCreateData.uuid)
await db.sequelize.transaction(async t => {
const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t)
if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.')
const videoAuthorData = {
name: authorToCreateData.name,
uuid: authorToCreateData.uuid,
userId: null, // Not on our pod
podId: fromPod.id
}
const author = db.Author.build(videoAuthorData)
await author.save({ transaction: t })
})
logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid)
}
async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
const options = {
arguments: [ authorAttributesToRemove, fromPod ],
errorMessage: 'Cannot remove the remote video author with many retries.'
}
await retryTransactionWrapper(removeRemoteVideoAuthor, options)
}
async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid)
await db.sequelize.transaction(async t => {
const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t)
await videoAuthor.destroy({ transaction: t })
})
logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid)
}
async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
const options = {
arguments: [ videoChannelToCreateData, fromPod ],
errorMessage: 'Cannot insert the remote video channel with many retries.'
}
await retryTransactionWrapper(addRemoteVideoChannel, options)
}
async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid)
await db.sequelize.transaction(async t => {
const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid)
if (videoChannelInDatabase) {
throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.')
}
const authorUUID = videoChannelToCreateData.ownerUUID
const podId = fromPod.id
const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t)
if (!author) throw new Error('Unknown author UUID' + authorUUID + '.')
const videoChannelData = {
name: videoChannelToCreateData.name,
description: videoChannelToCreateData.description,
uuid: videoChannelToCreateData.uuid,
createdAt: videoChannelToCreateData.createdAt,
updatedAt: videoChannelToCreateData.updatedAt,
remote: true,
authorId: author.id
}
const videoChannel = db.VideoChannel.build(videoChannelData)
await videoChannel.save({ transaction: t })
})
logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid)
}
async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
const options = {
arguments: [ videoChannelAttributesToUpdate, fromPod ],
errorMessage: 'Cannot update the remote video channel with many retries.'
}
await retryTransactionWrapper(updateRemoteVideoChannel, options)
}
async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid)
await db.sequelize.transaction(async t => {
const sequelizeOptions = { transaction: t }
const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t)
videoChannelInstance.set('name', videoChannelAttributesToUpdate.name)
videoChannelInstance.set('description', videoChannelAttributesToUpdate.description)
videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt)
videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt)
await videoChannelInstance.save(sequelizeOptions)
})
logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid)
}
async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
const options = {
arguments: [ videoChannelAttributesToRemove, fromPod ],
errorMessage: 'Cannot remove the remote video channel with many retries.'
}
await retryTransactionWrapper(removeRemoteVideoChannel, options)
}
async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid)
await db.sequelize.transaction(async t => {
const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t)
await videoChannel.destroy({ transaction: t })
})
logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid)
}
async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
const options = {
arguments: [ reportData, fromPod ],
errorMessage: 'Cannot create remote abuse video with many retries.'
}
await retryTransactionWrapper(reportAbuseRemoteVideo, options)
}
async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID)
await db.sequelize.transaction(async t => {
const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t)
const videoAbuseData = {
reporterUsername: reportData.reporterUsername,
reason: reportData.reportReason,
reporterPodId: fromPod.id,
videoId: videoInstance.id
}
await db.VideoAbuse.create(videoAbuseData)
})
logger.info('Remote abuse for video uuid %s created', reportData.videoUUID)
}
async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) {
try {
const video = await db.Video.loadLocalVideoByUUID(id, t)
if (!video) throw new Error('Video ' + id + ' not found')
return video
} catch (err) {
logger.error('Cannot load owned video from id.', { error: err.stack, id })
throw err
}
}
async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) {
try {
const video = await db.Video.loadByHostAndUUID(podHost, uuid, t)
if (!video) throw new Error('Video not found')
return video
} catch (err) {
logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid })
throw err
}
}

View File

@ -10,7 +10,8 @@ import {
VIDEO_CATEGORIES, VIDEO_CATEGORIES,
VIDEO_LICENCES, VIDEO_LICENCES,
VIDEO_LANGUAGES, VIDEO_LANGUAGES,
VIDEO_PRIVACIES VIDEO_PRIVACIES,
VIDEO_MIMETYPE_EXT
} from '../../../initializers' } from '../../../initializers'
import { import {
addEventToRemoteVideo, addEventToRemoteVideo,
@ -50,6 +51,7 @@ import { abuseVideoRouter } from './abuse'
import { blacklistRouter } from './blacklist' import { blacklistRouter } from './blacklist'
import { rateVideoRouter } from './rate' import { rateVideoRouter } from './rate'
import { videoChannelRouter } from './channel' import { videoChannelRouter } from './channel'
import { getActivityPubUrl } from '../../../helpers/activitypub'
const videosRouter = express.Router() const videosRouter = express.Router()
@ -59,19 +61,18 @@ const storage = multer.diskStorage({
cb(null, CONFIG.STORAGE.VIDEOS_DIR) cb(null, CONFIG.STORAGE.VIDEOS_DIR)
}, },
filename: (req, file, cb) => { filename: async (req, file, cb) => {
let extension = '' const extension = VIDEO_MIMETYPE_EXT[file.mimetype]
if (file.mimetype === 'video/webm') extension = 'webm' let randomString = ''
else if (file.mimetype === 'video/mp4') extension = 'mp4'
else if (file.mimetype === 'video/ogg') extension = 'ogv' try {
generateRandomString(16) randomString = await generateRandomString(16)
.then(randomString => { } catch (err) {
cb(null, randomString + '.' + extension) logger.error('Cannot generate random string for file name.', err)
}) randomString = 'fake-random-string'
.catch(err => { }
logger.error('Cannot generate random string for file name.', err)
throw err cb(null, randomString + '.' + extension)
})
} }
}) })
@ -190,6 +191,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi
channelId: res.locals.videoChannel.id channelId: res.locals.videoChannel.id
} }
const video = db.Video.build(videoData) const video = db.Video.build(videoData)
video.url = getActivityPubUrl('video', video.uuid)
const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename) const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename)
const videoFileHeight = await getVideoFileHeight(videoFilePath) const videoFileHeight = await getVideoFileHeight(videoFilePath)

View File

@ -2,10 +2,48 @@ import * as url from 'url'
import { database as db } from '../initializers' import { database as db } from '../initializers'
import { logger } from './logger' import { logger } from './logger'
import { doRequest } from './requests' import { doRequest, doRequestAndSaveToFile } from './requests'
import { isRemoteAccountValid } from './custom-validators' import { isRemoteAccountValid } from './custom-validators'
import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor'
import { ResultList } from '../../shared/models/result-list.model' import { ResultList } from '../../shared/models/result-list.model'
import { CONFIG } from '../initializers/constants'
import { VideoInstance } from '../models/video/video-interface'
import { ActivityIconObject } from '../../shared/index'
import { join } from 'path'
function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) {
const thumbnailName = video.getThumbnailName()
const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
const options = {
method: 'GET',
uri: icon.url
}
return doRequestAndSaveToFile(options, thumbnailPath)
}
function getActivityPubUrl (type: 'video' | 'videoChannel', uuid: string) {
if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + uuid
else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + uuid
return ''
}
async function getOrCreateAccount (accountUrl: string) {
let account = await db.Account.loadByUrl(accountUrl)
// We don't have this account in our database, fetch it on remote
if (!account) {
const { account } = await fetchRemoteAccountAndCreatePod(accountUrl)
if (!account) throw new Error('Cannot fetch remote account.')
// Save our new account in database
await account.save()
}
return account
}
async function fetchRemoteAccountAndCreatePod (accountUrl: string) { async function fetchRemoteAccountAndCreatePod (accountUrl: string) {
const options = { const options = {
@ -100,7 +138,10 @@ function activityPubCollectionPagination (url: string, page: number, result: Res
export { export {
fetchRemoteAccountAndCreatePod, fetchRemoteAccountAndCreatePod,
activityPubContextify, activityPubContextify,
activityPubCollectionPagination activityPubCollectionPagination,
getActivityPubUrl,
generateThumbnailFromUrl,
getOrCreateAccount
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -0,0 +1,34 @@
import * as validator from 'validator'
import {
isVideoChannelCreateActivityValid,
isVideoTorrentAddActivityValid,
isVideoTorrentUpdateActivityValid,
isVideoChannelUpdateActivityValid
} from './videos'
function isRootActivityValid (activity: any) {
return Array.isArray(activity['@context']) &&
(
(activity.type === 'Collection' || activity.type === 'OrderedCollection') &&
validator.isInt(activity.totalItems, { min: 0 }) &&
Array.isArray(activity.items)
) ||
(
validator.isURL(activity.id) &&
validator.isURL(activity.actor)
)
}
function isActivityValid (activity: any) {
return isVideoTorrentAddActivityValid(activity) ||
isVideoChannelCreateActivityValid(activity) ||
isVideoTorrentUpdateActivityValid(activity) ||
isVideoChannelUpdateActivityValid(activity)
}
// ---------------------------------------------------------------------------
export {
isRootActivityValid,
isActivityValid
}

View File

@ -1,4 +1,5 @@
export * from './account' export * from './account'
export * from './activity'
export * from './signature' export * from './signature'
export * from './misc' export * from './misc'
export * from './videos' export * from './videos'

View File

@ -12,6 +12,16 @@ function isActivityPubUrlValid (url: string) {
return exists(url) && validator.isURL(url, isURLOptions) return exists(url) && validator.isURL(url, isURLOptions)
} }
export { function isBaseActivityValid (activity: any, type: string) {
isActivityPubUrlValid return Array.isArray(activity['@context']) &&
activity.type === type &&
validator.isURL(activity.id) &&
validator.isURL(activity.actor) &&
Array.isArray(activity.to) &&
activity.to.every(t => validator.isURL(t))
}
export {
isActivityPubUrlValid,
isBaseActivityValid
} }

View File

@ -1,184 +1,117 @@
import 'express-validator' import * as validator from 'validator'
import { has, values } from 'lodash'
import { import {
REQUEST_ENDPOINTS, ACTIVITY_PUB
REQUEST_ENDPOINT_ACTIONS,
REQUEST_VIDEO_EVENT_TYPES
} from '../../../initializers' } from '../../../initializers'
import { isArray, isDateValid, isUUIDValid } from '../misc' import { isDateValid, isUUIDValid } from '../misc'
import { import {
isVideoThumbnailDataValid,
isVideoAbuseReasonValid,
isVideoAbuseReporterUsernameValid,
isVideoViewsValid, isVideoViewsValid,
isVideoLikesValid,
isVideoDislikesValid,
isVideoEventCountValid,
isRemoteVideoCategoryValid,
isRemoteVideoLicenceValid,
isRemoteVideoLanguageValid,
isVideoNSFWValid, isVideoNSFWValid,
isVideoTruncatedDescriptionValid, isVideoTruncatedDescriptionValid,
isVideoDurationValid, isVideoDurationValid,
isVideoFileInfoHashValid,
isVideoNameValid, isVideoNameValid,
isVideoTagsValid, isVideoTagValid
isVideoFileExtnameValid,
isVideoFileResolutionValid
} from '../videos' } from '../videos'
import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels' import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels'
import { isVideoAuthorNameValid } from '../video-authors' import { isBaseActivityValid } from './misc'
const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] function isVideoTorrentAddActivityValid (activity: any) {
return isBaseActivityValid(activity, 'Add') &&
const checkers: { [ id: string ]: (obj: any) => boolean } = {} isVideoTorrentObjectValid(activity.object)
checkers[ENDPOINT_ACTIONS.ADD_VIDEO] = checkAddVideo
checkers[ENDPOINT_ACTIONS.UPDATE_VIDEO] = checkUpdateVideo
checkers[ENDPOINT_ACTIONS.REMOVE_VIDEO] = checkRemoveVideo
checkers[ENDPOINT_ACTIONS.REPORT_ABUSE] = checkReportVideo
checkers[ENDPOINT_ACTIONS.ADD_CHANNEL] = checkAddVideoChannel
checkers[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = checkUpdateVideoChannel
checkers[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = checkRemoveVideoChannel
checkers[ENDPOINT_ACTIONS.ADD_AUTHOR] = checkAddAuthor
checkers[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = checkRemoveAuthor
function removeBadRequestVideos (requests: any[]) {
for (let i = requests.length - 1; i >= 0 ; i--) {
const request = requests[i]
const video = request.data
if (
!video ||
checkers[request.type] === undefined ||
checkers[request.type](video) === false
) {
requests.splice(i, 1)
}
}
} }
function removeBadRequestVideosQadu (requests: any[]) { function isVideoTorrentUpdateActivityValid (activity: any) {
for (let i = requests.length - 1; i >= 0 ; i--) { return isBaseActivityValid(activity, 'Update') &&
const request = requests[i] isVideoTorrentObjectValid(activity.object)
const video = request.data
if (
!video ||
(
isUUIDValid(video.uuid) &&
(has(video, 'views') === false || isVideoViewsValid(video.views)) &&
(has(video, 'likes') === false || isVideoLikesValid(video.likes)) &&
(has(video, 'dislikes') === false || isVideoDislikesValid(video.dislikes))
) === false
) {
requests.splice(i, 1)
}
}
} }
function removeBadRequestVideosEvents (requests: any[]) { function isVideoTorrentObjectValid (video: any) {
for (let i = requests.length - 1; i >= 0 ; i--) { return video.type === 'Video' &&
const request = requests[i] isVideoNameValid(video.name) &&
const eventData = request.data isVideoDurationValid(video.duration) &&
isUUIDValid(video.uuid) &&
setValidRemoteTags(video) &&
isRemoteIdentifierValid(video.category) &&
isRemoteIdentifierValid(video.licence) &&
isRemoteIdentifierValid(video.language) &&
isVideoViewsValid(video.video) &&
isVideoNSFWValid(video.nsfw) &&
isDateValid(video.published) &&
isDateValid(video.updated) &&
isRemoteVideoContentValid(video.mediaType, video.content) &&
isRemoteVideoIconValid(video.icon) &&
setValidRemoteVideoUrls(video.url)
}
if ( function isVideoChannelCreateActivityValid (activity: any) {
!eventData || return isBaseActivityValid(activity, 'Create') &&
( isVideoChannelObjectValid(activity.object)
isUUIDValid(eventData.uuid) && }
values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 &&
isVideoEventCountValid(eventData.count) function isVideoChannelUpdateActivityValid (activity: any) {
) === false return isBaseActivityValid(activity, 'Update') &&
) { isVideoChannelObjectValid(activity.object)
requests.splice(i, 1) }
}
} function isVideoChannelObjectValid (videoChannel: any) {
return videoChannel.type === 'VideoChannel' &&
isVideoChannelNameValid(videoChannel.name) &&
isVideoChannelDescriptionValid(videoChannel.description) &&
isUUIDValid(videoChannel.uuid)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export { export {
removeBadRequestVideos, isVideoTorrentAddActivityValid,
removeBadRequestVideosQadu, isVideoChannelCreateActivityValid,
removeBadRequestVideosEvents isVideoTorrentUpdateActivityValid,
isVideoChannelUpdateActivityValid
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function isCommonVideoAttributesValid (video: any) { function setValidRemoteTags (video: any) {
return isDateValid(video.createdAt) && if (Array.isArray(video.tag) === false) return false
isDateValid(video.updatedAt) &&
isRemoteVideoCategoryValid(video.category) &&
isRemoteVideoLicenceValid(video.licence) &&
isRemoteVideoLanguageValid(video.language) &&
isVideoNSFWValid(video.nsfw) &&
isVideoTruncatedDescriptionValid(video.truncatedDescription) &&
isVideoDurationValid(video.duration) &&
isVideoNameValid(video.name) &&
isVideoTagsValid(video.tags) &&
isUUIDValid(video.uuid) &&
isVideoViewsValid(video.views) &&
isVideoLikesValid(video.likes) &&
isVideoDislikesValid(video.dislikes) &&
isArray(video.files) &&
video.files.every(videoFile => {
if (!videoFile) return false
return ( const newTag = video.tag.filter(t => {
isVideoFileInfoHashValid(videoFile.infoHash) && return t.type === 'Hashtag' &&
isVideoFileExtnameValid(videoFile.extname) && isVideoTagValid(t.name)
isVideoFileResolutionValid(videoFile.resolution) })
)
}) video.tag = newTag
return true
} }
function checkAddVideo (video: any) { function isRemoteIdentifierValid (data: any) {
return isCommonVideoAttributesValid(video) && return validator.isInt(data.identifier, { min: 0 })
isUUIDValid(video.channelUUID) &&
isVideoThumbnailDataValid(video.thumbnailData)
} }
function checkUpdateVideo (video: any) { function isRemoteVideoContentValid (mediaType: string, content: string) {
return isCommonVideoAttributesValid(video) return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content)
} }
function checkRemoveVideo (video: any) { function isRemoteVideoIconValid (icon: any) {
return isUUIDValid(video.uuid) return icon.type === 'Image' &&
validator.isURL(icon.url) &&
icon.mediaType === 'image/jpeg' &&
validator.isInt(icon.width, { min: 0 }) &&
validator.isInt(icon.height, { min: 0 })
} }
function checkReportVideo (abuse: any) { function setValidRemoteVideoUrls (video: any) {
return isUUIDValid(abuse.videoUUID) && if (Array.isArray(video.url) === false) return false
isVideoAbuseReasonValid(abuse.reportReason) &&
isVideoAbuseReporterUsernameValid(abuse.reporterUsername) const newUrl = video.url.filter(u => isRemoteVideoUrlValid(u))
video.url = newUrl
return true
} }
function checkAddVideoChannel (videoChannel: any) { function isRemoteVideoUrlValid (url: any) {
return isUUIDValid(videoChannel.uuid) && return url.type === 'Link' &&
isVideoChannelNameValid(videoChannel.name) && ACTIVITY_PUB.VIDEO_URL_MIME_TYPES.indexOf(url.mimeType) !== -1 &&
isVideoChannelDescriptionValid(videoChannel.description) && validator.isURL(url.url) &&
isDateValid(videoChannel.createdAt) && validator.isInt(url.width, { min: 0 }) &&
isDateValid(videoChannel.updatedAt) && validator.isInt(url.size, { min: 0 })
isUUIDValid(videoChannel.ownerUUID)
}
function checkUpdateVideoChannel (videoChannel: any) {
return isUUIDValid(videoChannel.uuid) &&
isVideoChannelNameValid(videoChannel.name) &&
isVideoChannelDescriptionValid(videoChannel.description) &&
isDateValid(videoChannel.createdAt) &&
isDateValid(videoChannel.updatedAt) &&
isUUIDValid(videoChannel.ownerUUID)
}
function checkRemoveVideoChannel (videoChannel: any) {
return isUUIDValid(videoChannel.uuid)
}
function checkAddAuthor (author: any) {
return isUUIDValid(author.uuid) &&
isVideoAuthorNameValid(author.name)
}
function checkRemoveAuthor (author: any) {
return isUUIDValid(author.uuid)
} }

View File

@ -3,6 +3,5 @@ export * from './misc'
export * from './pods' export * from './pods'
export * from './pods' export * from './pods'
export * from './users' export * from './users'
export * from './video-authors'
export * from './video-channels' export * from './video-channels'
export * from './videos' export * from './videos'

View File

@ -1,45 +0,0 @@
import * as Promise from 'bluebird'
import * as validator from 'validator'
import * as express from 'express'
import 'express-validator'
import { database as db } from '../../initializers'
import { AuthorInstance } from '../../models'
import { logger } from '../logger'
import { isUserUsernameValid } from './users'
function isVideoAuthorNameValid (value: string) {
return isUserUsernameValid(value)
}
function checkVideoAuthorExists (id: string, res: express.Response, callback: () => void) {
let promise: Promise<AuthorInstance>
if (validator.isInt(id)) {
promise = db.Author.load(+id)
} else { // UUID
promise = db.Author.loadByUUID(id)
}
promise.then(author => {
if (!author) {
return res.status(404)
.json({ error: 'Video author not found' })
.end()
}
res.locals.author = author
callback()
})
.catch(err => {
logger.error('Error in video author request validator.', err)
return res.sendStatus(500)
})
}
// ---------------------------------------------------------------------------
export {
checkVideoAuthorExists,
isVideoAuthorNameValid
}

View File

@ -73,19 +73,26 @@ function isVideoDescriptionValid (value: string) {
} }
function isVideoDurationValid (value: string) { function isVideoDurationValid (value: string) {
return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
return exists(value) &&
typeof value === 'string' &&
value.startsWith('PT') &&
value.endsWith('S') &&
validator.isInt(value.replace(/[^0-9]+/, ''), VIDEOS_CONSTRAINTS_FIELDS.DURATION)
} }
function isVideoNameValid (value: string) { function isVideoNameValid (value: string) {
return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME)
} }
function isVideoTagValid (tag: string) {
return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG)
}
function isVideoTagsValid (tags: string[]) { function isVideoTagsValid (tags: string[]) {
return isArray(tags) && return isArray(tags) &&
validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) &&
tags.every(tag => { tags.every(tag => isVideoTagValid(tag))
return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG)
})
} }
function isVideoThumbnailValid (value: string) { function isVideoThumbnailValid (value: string) {
@ -209,6 +216,7 @@ export {
isRemoteVideoPrivacyValid, isRemoteVideoPrivacyValid,
isVideoFileResolutionValid, isVideoFileResolutionValid,
checkVideoExists, checkVideoExists,
isVideoTagValid,
isRemoteVideoCategoryValid, isRemoteVideoCategoryValid,
isRemoteVideoLicenceValid, isRemoteVideoLicenceValid,
isRemoteVideoLanguageValid isRemoteVideoLanguageValid

View File

@ -10,6 +10,7 @@ import {
import { PodInstance } from '../models' import { PodInstance } from '../models'
import { PodSignature } from '../../shared' import { PodSignature } from '../../shared'
import { signObject } from './peertube-crypto' import { signObject } from './peertube-crypto'
import { createWriteStream } from 'fs'
function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { function doRequest (requestOptions: request.CoreOptions & request.UriOptions) {
return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => {
@ -17,6 +18,15 @@ function doRequest (requestOptions: request.CoreOptions & request.UriOptions) {
}) })
} }
function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.UriOptions, destPath: string) {
return new Promise<request.RequestResponse>((res, rej) => {
request(requestOptions)
.on('response', response => res(response as request.RequestResponse))
.on('error', err => rej(err))
.pipe(createWriteStream(destPath))
})
}
type MakeRetryRequestParams = { type MakeRetryRequestParams = {
url: string, url: string,
method: 'GET' | 'POST', method: 'GET' | 'POST',
@ -88,6 +98,7 @@ function makeSecureRequest (params: MakeSecureRequestParams) {
export { export {
doRequest, doRequest,
doRequestAndSaveToFile,
makeRetryRequest, makeRetryRequest,
makeSecureRequest makeSecureRequest
} }

View File

@ -203,6 +203,12 @@ const VIDEO_PRIVACIES = {
[VideoPrivacy.PRIVATE]: 'Private' [VideoPrivacy.PRIVATE]: 'Private'
} }
const VIDEO_MIMETYPE_EXT = {
'video/webm': 'webm',
'video/ogg': 'ogv',
'video/mp4': 'mp4'
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Score a pod has when we create it as a friend // Score a pod has when we create it as a friend
@ -212,7 +218,14 @@ const FRIEND_SCORE = {
} }
const ACTIVITY_PUB = { const ACTIVITY_PUB = {
COLLECTION_ITEMS_PER_PAGE: 10 COLLECTION_ITEMS_PER_PAGE: 10,
VIDEO_URL_MIME_TYPES: [
'video/mp4',
'video/webm',
'video/ogg',
'application/x-bittorrent',
'application/x-bittorrent;x-scheme-handler/magnet'
]
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -245,42 +258,6 @@ const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50
// Number of requests to retry for replay requests module // Number of requests to retry for replay requests module
const RETRY_REQUESTS = 5 const RETRY_REQUESTS = 5
const REQUEST_ENDPOINTS: { [ id: string ]: RequestEndpoint } = {
VIDEOS: 'videos'
}
const REQUEST_ENDPOINT_ACTIONS: {
[ id: string ]: {
[ id: string ]: RemoteVideoRequestType
}
} = {}
REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
ADD_VIDEO: 'add-video',
UPDATE_VIDEO: 'update-video',
REMOVE_VIDEO: 'remove-video',
ADD_CHANNEL: 'add-channel',
UPDATE_CHANNEL: 'update-channel',
REMOVE_CHANNEL: 'remove-channel',
ADD_AUTHOR: 'add-author',
REMOVE_AUTHOR: 'remove-author',
REPORT_ABUSE: 'report-abuse'
}
const REQUEST_VIDEO_QADU_ENDPOINT = 'videos/qadu'
const REQUEST_VIDEO_EVENT_ENDPOINT = 'videos/events'
const REQUEST_VIDEO_QADU_TYPES: { [ id: string ]: RequestVideoQaduType } = {
LIKES: 'likes',
DISLIKES: 'dislikes',
VIEWS: 'views'
}
const REQUEST_VIDEO_EVENT_TYPES: { [ id: string ]: RequestVideoEventType } = {
LIKES: 'likes',
DISLIKES: 'dislikes',
VIEWS: 'views'
}
const REMOTE_SCHEME = { const REMOTE_SCHEME = {
HTTP: 'https', HTTP: 'https',
WS: 'wss' WS: 'wss'
@ -306,8 +283,6 @@ let JOBS_FETCHING_INTERVAL = 60000
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// const SIGNATURE_ALGORITHM = 'RSA-SHA256'
// const SIGNATURE_ENCODING = 'hex'
const PRIVATE_RSA_KEY_SIZE = 2048 const PRIVATE_RSA_KEY_SIZE = 2048
// Password encryption // Password encryption
@ -412,5 +387,6 @@ export {
VIDEO_LANGUAGES, VIDEO_LANGUAGES,
VIDEO_PRIVACIES, VIDEO_PRIVACIES,
VIDEO_LICENCES, VIDEO_LICENCES,
VIDEO_RATE_TYPES VIDEO_RATE_TYPES,
VIDEO_MIMETYPE_EXT
} }

View File

@ -0,0 +1,77 @@
import * as magnetUtil from 'magnet-uri'
import * as Sequelize from 'sequelize'
import { VideoTorrentObject } from '../../../shared'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { database as db } from '../../initializers'
import { VIDEO_MIMETYPE_EXT } from '../../initializers/constants'
import { VideoChannelInstance } from '../../models/video/video-channel-interface'
import { VideoFileAttributes } from '../../models/video/video-file-interface'
import { VideoAttributes, VideoInstance } from '../../models/video/video-interface'
async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelInstance, videoObject: VideoTorrentObject, t: Sequelize.Transaction) {
const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t)
if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.')
const duration = videoObject.duration.replace(/[^\d]+/, '')
const videoData: VideoAttributes = {
name: videoObject.name,
uuid: videoObject.uuid,
url: videoObject.id,
category: parseInt(videoObject.category.identifier, 10),
licence: parseInt(videoObject.licence.identifier, 10),
language: parseInt(videoObject.language.identifier, 10),
nsfw: videoObject.nsfw,
description: videoObject.content,
channelId: videoChannel.id,
duration: parseInt(duration, 10),
createdAt: videoObject.published,
// FIXME: updatedAt does not seems to be considered by Sequelize
updatedAt: videoObject.updated,
views: videoObject.views,
likes: 0,
dislikes: 0,
// likes: videoToCreateData.likes,
// dislikes: videoToCreateData.dislikes,
remote: true,
privacy: 1
// privacy: videoToCreateData.privacy
}
return videoData
}
function videoFileActivityUrlToDBAttributes (videoCreated: VideoInstance, videoObject: VideoTorrentObject) {
const fileUrls = videoObject.url
.filter(u => Object.keys(VIDEO_MIMETYPE_EXT).indexOf(u.mimeType) !== -1)
const attributes: VideoFileAttributes[] = []
for (const url of fileUrls) {
// Fetch associated magnet uri
const magnet = videoObject.url
.find(u => {
return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === url.width
})
if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + url.url)
const parsed = magnetUtil.decode(magnet.url)
if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url)
const attribute = {
extname: VIDEO_MIMETYPE_EXT[url.mimeType],
infoHash: parsed.infoHash,
resolution: url.width,
size: url.size,
videoId: videoCreated.id
}
attributes.push(attribute)
}
return attributes
}
// ---------------------------------------------------------------------------
export {
videoFileActivityUrlToDBAttributes,
videoActivityObjectToDBAttributes
}

View File

@ -0,0 +1,72 @@
import { VideoTorrentObject } from '../../../shared'
import { ActivityAdd } from '../../../shared/models/activitypub/activity'
import { generateThumbnailFromUrl, logger, retryTransactionWrapper, getOrCreateAccount } from '../../helpers'
import { database as db } from '../../initializers'
import { AccountInstance } from '../../models/account/account-interface'
import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
import Bluebird = require('bluebird')
async function processAddActivity (activity: ActivityAdd) {
const activityObject = activity.object
const activityType = activityObject.type
const account = await getOrCreateAccount(activity.actor)
if (activityType === 'Video') {
return processAddVideo(account, activity.id, activityObject as VideoTorrentObject)
}
logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
return Promise.resolve(undefined)
}
// ---------------------------------------------------------------------------
export {
processAddActivity
}
// ---------------------------------------------------------------------------
function processAddVideo (account: AccountInstance, videoChannelUrl: string, video: VideoTorrentObject) {
const options = {
arguments: [ account, videoChannelUrl ,video ],
errorMessage: 'Cannot insert the remote video with many retries.'
}
return retryTransactionWrapper(addRemoteVideo, options)
}
async function addRemoteVideo (account: AccountInstance, videoChannelUrl: string, videoToCreateData: VideoTorrentObject) {
logger.debug('Adding remote video %s.', videoToCreateData.url)
await db.sequelize.transaction(async t => {
const sequelizeOptions = {
transaction: t
}
const videoChannel = await db.VideoChannel.loadByUrl(videoChannelUrl, t)
if (!videoChannel) throw new Error('Video channel not found.')
if (videoChannel.Account.id !== account.id) throw new Error('Video channel is not owned by this account.')
const videoData = await videoActivityObjectToDBAttributes(videoChannel, videoToCreateData, t)
const video = db.Video.build(videoData)
// Don't block on request
generateThumbnailFromUrl(video, videoToCreateData.icon)
.catch(err => logger.warning('Cannot generate thumbnail of %s.', videoToCreateData.id, err))
const videoCreated = await video.save(sequelizeOptions)
const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData)
const tasks: Bluebird<any>[] = videoFileAttributes.map(f => db.VideoFile.create(f))
await Promise.all(tasks)
const tags = videoToCreateData.tag.map(t => t.name)
const tagInstances = await db.Tag.findOrCreateTags(tags, t)
await videoCreated.setTags(tagInstances, sequelizeOptions)
})
logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
}

View File

@ -1,23 +1,23 @@
import { import { ActivityCreate, VideoChannelObject, VideoTorrentObject } from '../../../shared'
ActivityCreate, import { ActivityAdd } from '../../../shared/models/activitypub/activity'
VideoTorrentObject, import { generateThumbnailFromUrl, logger, retryTransactionWrapper } from '../../helpers'
VideoChannelObject
} from '../../../shared'
import { database as db } from '../../initializers' import { database as db } from '../../initializers'
import { logger, retryTransactionWrapper } from '../../helpers' import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
import Bluebird = require('bluebird')
import { AccountInstance } from '../../models/account/account-interface'
import { getActivityPubUrl, getOrCreateAccount } from '../../helpers/activitypub'
function processCreateActivity (activity: ActivityCreate) { async function processCreateActivity (activity: ActivityCreate) {
const activityObject = activity.object const activityObject = activity.object
const activityType = activityObject.type const activityType = activityObject.type
const account = await getOrCreateAccount(activity.actor)
if (activityType === 'Video') { if (activityType === 'VideoChannel') {
return processCreateVideo(activityObject as VideoTorrentObject) return processCreateVideoChannel(account, activityObject as VideoChannelObject)
} else if (activityType === 'VideoChannel') {
return processCreateVideoChannel(activityObject as VideoChannelObject)
} }
logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
return Promise.resolve() return Promise.resolve(undefined)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -28,77 +28,37 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function processCreateVideo (video: VideoTorrentObject) { function processCreateVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) {
const options = { const options = {
arguments: [ video ], arguments: [ account, videoChannelToCreateData ],
errorMessage: 'Cannot insert the remote video with many retries.' errorMessage: 'Cannot insert the remote video channel with many retries.'
} }
return retryTransactionWrapper(addRemoteVideo, options) return retryTransactionWrapper(addRemoteVideoChannel, options)
} }
async function addRemoteVideo (videoToCreateData: VideoTorrentObject) { async function addRemoteVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) {
logger.debug('Adding remote video %s.', videoToCreateData.url) logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid)
await db.sequelize.transaction(async t => { await db.sequelize.transaction(async t => {
const sequelizeOptions = { let videoChannel = await db.VideoChannel.loadByUUIDOrUrl(videoChannelToCreateData.uuid, videoChannelToCreateData.id, t)
transaction: t if (videoChannel) throw new Error('Video channel with this URL/UUID already exists.')
}
const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) const videoChannelData = {
if (videoFromDatabase) throw new Error('UUID already exists.') name: videoChannelToCreateData.name,
description: videoChannelToCreateData.content,
const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) uuid: videoChannelToCreateData.uuid,
if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') createdAt: videoChannelToCreateData.published,
updatedAt: videoChannelToCreateData.updated,
const tags = videoToCreateData.tags
const tagInstances = await db.Tag.findOrCreateTags(tags, t)
const videoData = {
name: videoToCreateData.name,
uuid: videoToCreateData.uuid,
category: videoToCreateData.category,
licence: videoToCreateData.licence,
language: videoToCreateData.language,
nsfw: videoToCreateData.nsfw,
description: videoToCreateData.truncatedDescription,
channelId: videoChannel.id,
duration: videoToCreateData.duration,
createdAt: videoToCreateData.createdAt,
// FIXME: updatedAt does not seems to be considered by Sequelize
updatedAt: videoToCreateData.updatedAt,
views: videoToCreateData.views,
likes: videoToCreateData.likes,
dislikes: videoToCreateData.dislikes,
remote: true, remote: true,
privacy: videoToCreateData.privacy accountId: account.id
} }
const video = db.Video.build(videoData) videoChannel = db.VideoChannel.build(videoChannelData)
await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) videoChannel.url = getActivityPubUrl('videoChannel', videoChannel.uuid)
const videoCreated = await video.save(sequelizeOptions)
const tasks = [] await videoChannel.save({ transaction: t })
for (const fileData of videoToCreateData.files) {
const videoFileInstance = db.VideoFile.build({
extname: fileData.extname,
infoHash: fileData.infoHash,
resolution: fileData.resolution,
size: fileData.size,
videoId: videoCreated.id
})
tasks.push(videoFileInstance.save(sequelizeOptions))
}
await Promise.all(tasks)
await videoCreated.setTags(tagInstances, sequelizeOptions)
}) })
logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid)
}
function processCreateVideoChannel (videoChannel: VideoChannelObject) {
} }

View File

@ -1,15 +1,25 @@
import { import { VideoChannelObject, VideoTorrentObject } from '../../../shared'
ActivityCreate, import { ActivityUpdate } from '../../../shared/models/activitypub/activity'
VideoTorrentObject, import { getOrCreateAccount } from '../../helpers/activitypub'
VideoChannelObject import { retryTransactionWrapper } from '../../helpers/database-utils'
} from '../../../shared' import { logger } from '../../helpers/logger'
import { resetSequelizeInstance } from '../../helpers/utils'
import { database as db } from '../../initializers'
import { AccountInstance } from '../../models/account/account-interface'
import { VideoInstance } from '../../models/video/video-interface'
import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
import Bluebird = require('bluebird')
async function processUpdateActivity (activity: ActivityUpdate) {
const account = await getOrCreateAccount(activity.actor)
function processUpdateActivity (activity: ActivityCreate) {
if (activity.object.type === 'Video') { if (activity.object.type === 'Video') {
return processUpdateVideo(activity.object) return processUpdateVideo(account, activity.object)
} else if (activity.object.type === 'VideoChannel') { } else if (activity.object.type === 'VideoChannel') {
return processUpdateVideoChannel(activity.object) return processUpdateVideoChannel(account, activity.object)
} }
return undefined
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -20,10 +30,107 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function processUpdateVideo (video: VideoTorrentObject) { function processUpdateVideo (account: AccountInstance, video: VideoTorrentObject) {
const options = {
arguments: [ account, video ],
errorMessage: 'Cannot update the remote video with many retries'
}
return retryTransactionWrapper(updateRemoteVideo, options)
} }
function processUpdateVideoChannel (videoChannel: VideoChannelObject) { async function updateRemoteVideo (account: AccountInstance, videoAttributesToUpdate: VideoTorrentObject) {
logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
let videoInstance: VideoInstance
let videoFieldsSave: object
try {
await db.sequelize.transaction(async t => {
const sequelizeOptions = {
transaction: t
}
const videoInstance = await db.Video.loadByUrl(videoAttributesToUpdate.id, t)
if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.')
if (videoInstance.VideoChannel.Account.id !== account.id) {
throw new Error('Account ' + account.url + ' does not own video channel ' + videoInstance.VideoChannel.url)
}
const videoData = await videoActivityObjectToDBAttributes(videoInstance.VideoChannel, videoAttributesToUpdate, t)
videoInstance.set('name', videoData.name)
videoInstance.set('category', videoData.category)
videoInstance.set('licence', videoData.licence)
videoInstance.set('language', videoData.language)
videoInstance.set('nsfw', videoData.nsfw)
videoInstance.set('description', videoData.description)
videoInstance.set('duration', videoData.duration)
videoInstance.set('createdAt', videoData.createdAt)
videoInstance.set('updatedAt', videoData.updatedAt)
videoInstance.set('views', videoData.views)
// videoInstance.set('likes', videoData.likes)
// videoInstance.set('dislikes', videoData.dislikes)
// videoInstance.set('privacy', videoData.privacy)
await videoInstance.save(sequelizeOptions)
// Remove old video files
const videoFileDestroyTasks: Bluebird<void>[] = []
for (const videoFile of videoInstance.VideoFiles) {
videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
}
await Promise.all(videoFileDestroyTasks)
const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoInstance, videoAttributesToUpdate)
const tasks: Bluebird<any>[] = videoFileAttributes.map(f => db.VideoFile.create(f))
await Promise.all(tasks)
const tags = videoAttributesToUpdate.tag.map(t => t.name)
const tagInstances = await db.Tag.findOrCreateTags(tags, t)
await videoInstance.setTags(tagInstances, sequelizeOptions)
})
logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)
} catch (err) {
if (videoInstance !== undefined && videoFieldsSave !== undefined) {
resetSequelizeInstance(videoInstance, videoFieldsSave)
}
// This is just a debug because we will retry the insert
logger.debug('Cannot update the remote video.', err)
throw err
}
}
async function processUpdateVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) {
const options = {
arguments: [ account, videoChannel ],
errorMessage: 'Cannot update the remote video channel with many retries.'
}
await retryTransactionWrapper(updateRemoteVideoChannel, options)
}
async function updateRemoteVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) {
logger.debug('Updating remote video channel "%s".', videoChannel.uuid)
await db.sequelize.transaction(async t => {
const sequelizeOptions = { transaction: t }
const videoChannelInstance = await db.VideoChannel.loadByUrl(videoChannel.id)
if (!videoChannelInstance) throw new Error('Video ' + videoChannel.id + ' not found.')
if (videoChannelInstance.Account.id !== account.id) {
throw new Error('Account ' + account.id + ' does not own video channel ' + videoChannelInstance.url)
}
videoChannelInstance.set('name', videoChannel.name)
videoChannelInstance.set('description', videoChannel.content)
videoChannelInstance.set('createdAt', videoChannel.published)
videoChannelInstance.set('updatedAt', videoChannel.updated)
await videoChannelInstance.save(sequelizeOptions)
})
logger.info('Remote video channel with uuid %s updated', videoChannel.uuid)
} }

View File

@ -0,0 +1,21 @@
import { body } from 'express-validator/check'
import * as express from 'express'
import { logger, isRootActivityValid } from '../../../helpers'
import { checkErrors } from '../utils'
const activityPubValidator = [
body('data').custom(isRootActivityValid),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking activity pub parameters', { parameters: req.body })
checkErrors(req, res, next)
}
]
// ---------------------------------------------------------------------------
export {
activityPubValidator
}

View File

@ -1,61 +0,0 @@
import { body } from 'express-validator/check'
import * as express from 'express'
import {
logger,
isArray,
removeBadRequestVideos,
removeBadRequestVideosQadu,
removeBadRequestVideosEvents
} from '../../../helpers'
import { checkErrors } from '../utils'
const remoteVideosValidator = [
body('data').custom(isArray),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking remoteVideos parameters', { parameters: req.body })
checkErrors(req, res, () => {
removeBadRequestVideos(req.body.data)
return next()
})
}
]
const remoteQaduVideosValidator = [
body('data').custom(isArray),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body })
checkErrors(req, res, () => {
removeBadRequestVideosQadu(req.body.data)
return next()
})
}
]
const remoteEventsVideosValidator = [
body('data').custom(isArray),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body })
checkErrors(req, res, () => {
removeBadRequestVideosEvents(req.body.data)
return next()
})
}
]
// ---------------------------------------------------------------------------
export {
remoteVideosValidator,
remoteQaduVideosValidator,
remoteEventsVideosValidator
}

View File

@ -24,6 +24,8 @@ export namespace VideoChannelMethods {
export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance> export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance>
export type LoadByHostAndUUID = (uuid: string, podHost: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance> export type LoadByHostAndUUID = (uuid: string, podHost: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance>
export type LoadAndPopulateAccountAndVideos = (id: number) => Promise<VideoChannelInstance> export type LoadAndPopulateAccountAndVideos = (id: number) => Promise<VideoChannelInstance>
export type LoadByUrl = (uuid: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance>
export type LoadByUUIDOrUrl = (uuid: string, url: string, t?: Sequelize.Transaction) => Promise<VideoChannelInstance>
} }
export interface VideoChannelClass { export interface VideoChannelClass {
@ -37,6 +39,8 @@ export interface VideoChannelClass {
loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount
loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount
loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos
loadByUrl: VideoChannelMethods.LoadByUrl
loadByUUIDOrUrl: VideoChannelMethods.LoadByUUIDOrUrl
} }
export interface VideoChannelAttributes { export interface VideoChannelAttributes {
@ -45,7 +49,7 @@ export interface VideoChannelAttributes {
name: string name: string
description: string description: string
remote: boolean remote: boolean
url: string url?: string
Account?: AccountInstance Account?: AccountInstance
Videos?: VideoInstance[] Videos?: VideoInstance[]

View File

@ -25,6 +25,8 @@ let loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount
let loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount let loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount
let loadByHostAndUUID: VideoChannelMethods.LoadByHostAndUUID let loadByHostAndUUID: VideoChannelMethods.LoadByHostAndUUID
let loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos let loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos
let loadByUrl: VideoChannelMethods.LoadByUrl
let loadByUUIDOrUrl: VideoChannelMethods.LoadByUUIDOrUrl
export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) {
VideoChannel = sequelize.define<VideoChannelInstance, VideoChannelAttributes>('VideoChannel', VideoChannel = sequelize.define<VideoChannelInstance, VideoChannelAttributes>('VideoChannel',
@ -94,12 +96,14 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
loadByUUID, loadByUUID,
loadByHostAndUUID, loadByHostAndUUID,
loadAndPopulateAccountAndVideos, loadAndPopulateAccountAndVideos,
countByAccount countByAccount,
loadByUrl,
loadByUUIDOrUrl
] ]
const instanceMethods = [ const instanceMethods = [
isOwned, isOwned,
toFormattedJSON, toFormattedJSON,
toActivityPubObject, toActivityPubObject
] ]
addMethodsToModel(VideoChannel, classMethods, instanceMethods) addMethodsToModel(VideoChannel, classMethods, instanceMethods)
@ -254,6 +258,33 @@ loadByUUID = function (uuid: string, t?: Sequelize.Transaction) {
return VideoChannel.findOne(query) return VideoChannel.findOne(query)
} }
loadByUrl = function (url: string, t?: Sequelize.Transaction) {
const query: Sequelize.FindOptions<VideoChannelAttributes> = {
where: {
url
}
}
if (t !== undefined) query.transaction = t
return VideoChannel.findOne(query)
}
loadByUUIDOrUrl = function (uuid: string, url: string, t?: Sequelize.Transaction) {
const query: Sequelize.FindOptions<VideoChannelAttributes> = {
where: {
[Sequelize.Op.or]: [
{ uuid },
{ url }
]
},
}
if (t !== undefined) query.transaction = t
return VideoChannel.findOne(query)
}
loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Transaction) { loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Transaction) {
const query: Sequelize.FindOptions<VideoChannelAttributes> = { const query: Sequelize.FindOptions<VideoChannelAttributes> = {
where: { where: {

View File

@ -69,6 +69,7 @@ export namespace VideoMethods {
export type LoadAndPopulateAccount = (id: number) => Bluebird<VideoInstance> export type LoadAndPopulateAccount = (id: number) => Bluebird<VideoInstance>
export type LoadAndPopulateAccountAndPodAndTags = (id: number) => Bluebird<VideoInstance> export type LoadAndPopulateAccountAndPodAndTags = (id: number) => Bluebird<VideoInstance>
export type LoadByUUIDAndPopulateAccountAndPodAndTags = (uuid: string) => Bluebird<VideoInstance> export type LoadByUUIDAndPopulateAccountAndPodAndTags = (uuid: string) => Bluebird<VideoInstance>
export type LoadByUUIDOrURL = (uuid: string, url: string, t?: Sequelize.Transaction) => Bluebird<VideoInstance>
export type RemoveThumbnail = (this: VideoInstance) => Promise<void> export type RemoveThumbnail = (this: VideoInstance) => Promise<void>
export type RemovePreview = (this: VideoInstance) => Promise<void> export type RemovePreview = (this: VideoInstance) => Promise<void>
@ -89,6 +90,7 @@ export interface VideoClass {
loadByHostAndUUID: VideoMethods.LoadByHostAndUUID loadByHostAndUUID: VideoMethods.LoadByHostAndUUID
loadByUUID: VideoMethods.LoadByUUID loadByUUID: VideoMethods.LoadByUUID
loadByUrl: VideoMethods.LoadByUrl loadByUrl: VideoMethods.LoadByUrl
loadByUUIDOrURL: VideoMethods.LoadByUUIDOrURL
loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID
loadByUUIDAndPopulateAccountAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAccountAndPodAndTags loadByUUIDAndPopulateAccountAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAccountAndPodAndTags
searchAndPopulateAccountAndPodAndTags: VideoMethods.SearchAndPopulateAccountAndPodAndTags searchAndPopulateAccountAndPodAndTags: VideoMethods.SearchAndPopulateAccountAndPodAndTags
@ -109,7 +111,10 @@ export interface VideoAttributes {
likes?: number likes?: number
dislikes?: number dislikes?: number
remote: boolean remote: boolean
url: string url?: string
createdAt?: Date
updatedAt?: Date
parentId?: number parentId?: number
channelId?: number channelId?: number
@ -120,9 +125,6 @@ export interface VideoAttributes {
} }
export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.Instance<VideoAttributes> { export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.Instance<VideoAttributes> {
createdAt: Date
updatedAt: Date
createPreview: VideoMethods.CreatePreview createPreview: VideoMethods.CreatePreview
createThumbnail: VideoMethods.CreateThumbnail createThumbnail: VideoMethods.CreateThumbnail
createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash
@ -158,4 +160,3 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In
} }
export interface VideoModel extends VideoClass, Sequelize.Model<VideoInstance, VideoAttributes> {} export interface VideoModel extends VideoClass, Sequelize.Model<VideoInstance, VideoAttributes> {}

View File

@ -25,7 +25,8 @@ import {
statPromise, statPromise,
generateImageFromVideoFile, generateImageFromVideoFile,
transcode, transcode,
getVideoFileHeight getVideoFileHeight,
getActivityPubUrl
} from '../../helpers' } from '../../helpers'
import { import {
CONFIG, CONFIG,
@ -88,7 +89,7 @@ let listOwnedAndPopulateAccountAndTags: VideoMethods.ListOwnedAndPopulateAccount
let listOwnedByAccount: VideoMethods.ListOwnedByAccount let listOwnedByAccount: VideoMethods.ListOwnedByAccount
let load: VideoMethods.Load let load: VideoMethods.Load
let loadByUUID: VideoMethods.LoadByUUID let loadByUUID: VideoMethods.LoadByUUID
let loadByUrl: VideoMethods.LoadByUrl let loadByUUIDOrURL: VideoMethods.LoadByUUIDOrURL
let loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID let loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID
let loadAndPopulateAccount: VideoMethods.LoadAndPopulateAccount let loadAndPopulateAccount: VideoMethods.LoadAndPopulateAccount
let loadAndPopulateAccountAndPodAndTags: VideoMethods.LoadAndPopulateAccountAndPodAndTags let loadAndPopulateAccountAndPodAndTags: VideoMethods.LoadAndPopulateAccountAndPodAndTags
@ -277,6 +278,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
loadAndPopulateAccount, loadAndPopulateAccount,
loadAndPopulateAccountAndPodAndTags, loadAndPopulateAccountAndPodAndTags,
loadByHostAndUUID, loadByHostAndUUID,
loadByUUIDOrURL,
loadByUUID, loadByUUID,
loadLocalVideoByUUID, loadLocalVideoByUUID,
loadByUUIDAndPopulateAccountAndPodAndTags, loadByUUIDAndPopulateAccountAndPodAndTags,
@ -595,6 +597,7 @@ toActivityPubObject = function (this: VideoInstance) {
const videoObject: VideoTorrentObject = { const videoObject: VideoTorrentObject = {
type: 'Video', type: 'Video',
id: getActivityPubUrl('video', this.uuid),
name: this.name, name: this.name,
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
duration: 'PT' + this.duration + 'S', duration: 'PT' + this.duration + 'S',
@ -731,6 +734,7 @@ getCategoryLabel = function (this: VideoInstance) {
getLicenceLabel = function (this: VideoInstance) { getLicenceLabel = function (this: VideoInstance) {
let licenceLabel = VIDEO_LICENCES[this.licence] let licenceLabel = VIDEO_LICENCES[this.licence]
// Maybe our pod is not up to date and there are new licences since our version // Maybe our pod is not up to date and there are new licences since our version
if (!licenceLabel) licenceLabel = 'Unknown' if (!licenceLabel) licenceLabel = 'Unknown'
@ -946,6 +950,22 @@ loadByUUID = function (uuid: string, t?: Sequelize.Transaction) {
return Video.findOne(query) return Video.findOne(query)
} }
loadByUUIDOrURL = function (uuid: string, url: string, t?: Sequelize.Transaction) {
const query: Sequelize.FindOptions<VideoAttributes> = {
where: {
[Sequelize.Op.or]: [
{ uuid },
{ url }
]
},
include: [ Video['sequelize'].models.VideoFile ]
}
if (t !== undefined) query.transaction = t
return Video.findOne(query)
}
loadLocalVideoByUUID = function (uuid: string, t?: Sequelize.Transaction) { loadLocalVideoByUUID = function (uuid: string, t?: Sequelize.Transaction) {
const query: Sequelize.FindOptions<VideoAttributes> = { const query: Sequelize.FindOptions<VideoAttributes> = {
where: { where: {

View File

@ -7,7 +7,7 @@ import { ActivityPubSignature } from './activitypub-signature'
export type Activity = ActivityCreate | ActivityUpdate | ActivityFlag export type Activity = ActivityCreate | ActivityUpdate | ActivityFlag
// Flag -> report abuse // Flag -> report abuse
export type ActivityType = 'Create' | 'Update' | 'Flag' export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag'
export interface BaseActivity { export interface BaseActivity {
'@context'?: any[] '@context'?: any[]
@ -20,7 +20,12 @@ export interface BaseActivity {
export interface ActivityCreate extends BaseActivity { export interface ActivityCreate extends BaseActivity {
type: 'Create' type: 'Create'
object: VideoTorrentObject | VideoChannelObject object: VideoChannelObject
}
export interface ActivityAdd extends BaseActivity {
type: 'Add'
object: VideoTorrentObject
} }
export interface ActivityUpdate extends BaseActivity { export interface ActivityUpdate extends BaseActivity {

View File

@ -2,7 +2,10 @@ import { ActivityIdentifierObject } from './common-objects'
export interface VideoChannelObject { export interface VideoChannelObject {
type: 'VideoChannel' type: 'VideoChannel'
id: string
name: string name: string
content: string content: string
uuid: ActivityIdentifierObject uuid: string
published: Date
updated: Date
} }

View File

@ -7,6 +7,7 @@ import {
export interface VideoTorrentObject { export interface VideoTorrentObject {
type: 'Video' type: 'Video'
id: string
name: string name: string
duration: string duration: string
uuid: string uuid: string