interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Automatically remove bad followings

This commit is contained in:
Chocobozzz 2019-08-06 17:19:53 +02:00
parent 466e3f20a5
commit 6b9c966f64
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
14 changed files with 307 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
server/lib/activitypub/actor.ts
View File

@ -254,14 +254,14 @@ async function refreshActorIfNeeded (
await actor.save({ transaction: t })
if (actor.Account) {
actor.Account.set('name', result.name)
actor.Account.set('description', result.summary)
actor.Account.name = result.name
actor.Account.description = result.summary
await actor.Account.save({ transaction: t })
} else if (actor.VideoChannel) {
actor.VideoChannel.set('name', result.name)
actor.VideoChannel.set('description', result.summary)
actor.VideoChannel.set('support', result.support)
actor.VideoChannel.name = result.name
actor.VideoChannel.description = result.summary
actor.VideoChannel.support = result.support
await actor.VideoChannel.save({ transaction: t })
}

11
server/lib/activitypub/process/process-create.ts
View File

@ -4,7 +4,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { logger } from '../../../helpers/logger'
import { sequelizeTypescript } from '../../../initializers'
import { ActorModel } from '../../../models/activitypub/actor'
import { addVideoComment, resolveThread } from '../video-comments'
import { resolveThread } from '../video-comments'
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
import { forwardVideoRelatedActivity } from '../send/utils'
import { createOrUpdateCacheFile } from '../cache-file'
@ -13,6 +13,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
import { createOrUpdateVideoPlaylist } from '../playlist'
import { VideoModel } from '../../../models/video/video'
import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
import { VideoCommentModel } from '../../../models/video/video-comment'
async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
const { activity, byActor } = options
@ -83,9 +84,13 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url)
let video: VideoModel
let created: boolean
let comment: VideoCommentModel
try {
const resolveThreadResult = await resolveThread(commentObject.inReplyTo)
const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
video = resolveThreadResult.video
created = resolveThreadResult.commentCreated
comment = resolveThreadResult.comment
} catch (err) {
logger.debug(
'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.',
@ -95,8 +100,6 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
return
}
const { comment, created } = await addVideoComment(video, commentObject.id)
if (video.isOwned() && created === true) {
// Don't resend the activity to the sender
const exceptions = [ byActor ]

244
server/lib/activitypub/video-comments.ts
View File

@ -1,9 +1,7 @@
import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object'
import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
import { logger } from '../../helpers/logger'
import { doRequest } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { ActorModel } from '../../models/activitypub/actor'
import { VideoModel } from '../../models/video/video'
import { VideoCommentModel } from '../../models/video/video-comment'
import { getOrCreateActorAndServerAndModel } from './actor'
@ -11,79 +9,53 @@ import { getOrCreateVideoAndAccountAndChannel } from './videos'
import * as Bluebird from 'bluebird'
import { checkUrlsSameHost } from '../../helpers/activitypub'
async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
let originCommentId: number = null
let inReplyToCommentId: number = null
// If this is not a reply to the video (thread), create or get the parent comment
if (video.url !== comment.inReplyTo) {
const { comment: parent } = await addVideoComment(video, comment.inReplyTo)
if (!parent) {
logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
return undefined
}
originCommentId = parent.originCommentId || parent.id
inReplyToCommentId = parent.id
}
return {
url: comment.id,
text: comment.content,
videoId: video.id,
accountId: actor.Account.id,
inReplyToCommentId,
originCommentId,
createdAt: new Date(comment.published)
}
type ResolveThreadParams = {
url: string,
comments?: VideoCommentModel[],
isVideo?: boolean,
commentCreated?: boolean
}
type ResolveThreadResult = Promise<{ video: VideoModel, comment: VideoCommentModel, commentCreated: boolean }>
async function addVideoComments (commentUrls: string[], instance: VideoModel) {
async function addVideoComments (commentUrls: string[]) {
return Bluebird.map(commentUrls, commentUrl => {
return addVideoComment(instance, commentUrl)
return resolveThread({ url: commentUrl, isVideo: false })
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
}
async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
logger.info('Fetching remote video comment %s.', commentUrl)
async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
const { url, isVideo } = params
if (params.commentCreated === undefined) params.commentCreated = false
if (params.comments === undefined) params.comments = []
const { body } = await doRequest({
uri: commentUrl,
json: true,
activityPub: true
})
if (sanitizeAndCheckVideoCommentObject(body) === false) {
logger.debug('Remote video comment JSON %s is not valid.', commentUrl, { body })
return { created: false }
// Already have this comment?
if (isVideo !== true) {
const result = await resolveCommentFromDB(params)
if (result) return result
}
const actorUrl = body.attributedTo
if (!actorUrl) return { created: false }
try {
if (isVideo !== false) return await tryResolveThreadFromVideo(params)
if (checkUrlsSameHost(commentUrl, actorUrl) !== true) {
throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`)
return resolveParentComment(params)
} catch (err) {
logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err })
return resolveParentComment(params)
}
if (checkUrlsSameHost(body.id, commentUrl) !== true) {
throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`)
}
const actor = await getOrCreateActorAndServerAndModel(actorUrl, 'all')
const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
if (!entry) return { created: false }
const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true })
comment.Account = actor.Account
comment.Video = videoInstance
return { comment, created }
}
type ResolveThreadResult = Promise<{ video: VideoModel, parents: VideoCommentModel[] }>
async function resolveThread (url: string, comments: VideoCommentModel[] = []): ResolveThreadResult {
// Already have this comment?
const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url)
export {
addVideoComments,
resolveThread
}
// ---------------------------------------------------------------------------
async function resolveCommentFromDB (params: ResolveThreadParams) {
const { url, comments, commentCreated } = params
const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
if (commentFromDatabase) {
let parentComments = comments.concat([ commentFromDatabase ])
@ -94,79 +66,97 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []):
parentComments = parentComments.concat(data)
}
return resolveThread(commentFromDatabase.Video.url, parentComments)
return resolveThread({
url: commentFromDatabase.Video.url,
comments: parentComments,
isVideo: true,
commentCreated
})
}
try {
// Maybe it's a reply to a video?
// If yes, it's done: we resolved all the thread
const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
return undefined
}
if (comments.length !== 0) {
const firstReply = comments[ comments.length - 1 ]
firstReply.inReplyToCommentId = null
firstReply.originCommentId = null
firstReply.videoId = video.id
comments[comments.length - 1] = await firstReply.save()
async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
const { url, comments, commentCreated } = params
for (let i = comments.length - 2; i >= 0; i--) {
const comment = comments[ i ]
comment.originCommentId = firstReply.id
comment.inReplyToCommentId = comments[ i + 1 ].id
comment.videoId = video.id
// Maybe it's a reply to a video?
// If yes, it's done: we resolved all the thread
const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false }
const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
comments[i] = await comment.save()
}
let resultComment: VideoCommentModel
if (comments.length !== 0) {
const firstReply = comments[ comments.length - 1 ]
firstReply.inReplyToCommentId = null
firstReply.originCommentId = null
firstReply.videoId = video.id
firstReply.changed('updatedAt', true)
firstReply.Video = video
comments[comments.length - 1] = await firstReply.save()
for (let i = comments.length - 2; i >= 0; i--) {
const comment = comments[ i ]
comment.originCommentId = firstReply.id
comment.inReplyToCommentId = comments[ i + 1 ].id
comment.videoId = video.id
comment.changed('updatedAt', true)
comment.Video = video
comments[i] = await comment.save()
}
return { video, parents: comments }
} catch (err) {
logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err })
if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
throw new Error('Recursion limit reached when resolving a thread')
}
const { body } = await doRequest({
uri: url,
json: true,
activityPub: true
})
if (sanitizeAndCheckVideoCommentObject(body) === false) {
throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body))
}
const actorUrl = body.attributedTo
if (!actorUrl) throw new Error('Miss attributed to in comment')
if (checkUrlsSameHost(url, actorUrl) !== true) {
throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
}
if (checkUrlsSameHost(body.id, url) !== true) {
throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
}
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
const comment = new VideoCommentModel({
url: body.id,
text: body.content,
videoId: null,
accountId: actor.Account.id,
inReplyToCommentId: null,
originCommentId: null,
createdAt: new Date(body.published),
updatedAt: new Date(body.updated)
})
return resolveThread(body.inReplyTo, comments.concat([ comment ]))
resultComment = comments[0]
}
return { video, comment: resultComment, commentCreated }
}
export {
videoCommentActivityObjectToDBAttributes,
addVideoComments,
addVideoComment,
resolveThread
async function resolveParentComment (params: ResolveThreadParams) {
const { url, comments } = params
if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
throw new Error('Recursion limit reached when resolving a thread')
}
const { body } = await doRequest({
uri: url,
json: true,
activityPub: true
})
if (sanitizeAndCheckVideoCommentObject(body) === false) {
throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body))
}
const actorUrl = body.attributedTo
if (!actorUrl) throw new Error('Miss attributed to in comment')
if (checkUrlsSameHost(url, actorUrl) !== true) {
throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
}
if (checkUrlsSameHost(body.id, url) !== true) {
throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
}
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
const comment = new VideoCommentModel({
url: body.id,
text: body.content,
videoId: null,
accountId: actor.Account.id,
inReplyToCommentId: null,
originCommentId: null,
createdAt: new Date(body.published),
updatedAt: new Date(body.updated)
})
comment.Account = actor.Account
return resolveThread({
url: body.inReplyTo,
comments: comments.concat([ comment ]),
commentCreated: true
})
}

16
server/lib/activitypub/videos.ts
View File

@ -56,6 +56,7 @@ import { join } from 'path'
import { FilteredModelAttributes } from '../../typings/sequelize'
import { Hooks } from '../plugins/hooks'
import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
import { ActorFollowScoreCache } from '../files-cache'
async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
if (
@ -182,7 +183,7 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
}
if (syncParam.comments === true) {
const handler = items => addVideoComments(items, video)
const handler = items => addVideoComments(items)
const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
@ -421,10 +422,14 @@ async function refreshVideoIfNeeded (options: {
await retryTransactionWrapper(updateVideoFromAP, updateOptions)
await syncVideoExternalAttributes(video, videoObject, options.syncParam)
ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
return video
} catch (err) {
logger.warn('Cannot refresh video %s.', options.video.url, { err })
ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
// Don't refresh in loop
await video.setAsRefreshed()
return video
@ -500,7 +505,7 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
await Promise.all(playlistPromises)
const streamingPlaylists = await Promise.all(playlistPromises)
// Process tags
const tags = videoObject.tag
@ -513,7 +518,12 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
})
await Promise.all(videoCaptionsPromises)
const captions = await Promise.all(videoCaptionsPromises)
video.VideoFiles = videoFiles
video.VideoStreamingPlaylists = streamingPlaylists
video.Tags = tagInstances
video.VideoCaptions = captions
const autoBlacklisted = await autoBlacklistVideoIfNeeded({
video,

28
server/lib/files-cache/actor-follow-score-cache.ts
View File

@ -7,6 +7,8 @@ class ActorFollowScoreCache {
private static instance: ActorFollowScoreCache
private pendingFollowsScore: { [ url: string ]: number } = {}
private pendingBadServer = new Set<number>()
private pendingGoodServer = new Set<number>()
private constructor () {}
@ -32,7 +34,31 @@ class ActorFollowScoreCache {
}
}
getPendingFollowsScoreCopy () {
addBadServerId (serverId: number) {
this.pendingBadServer.add(serverId)
}
getBadFollowingServerIds () {
return Array.from(this.pendingBadServer)
}
clearBadFollowingServerIds () {
this.pendingBadServer = new Set<number>()
}
addGoodServerId (serverId: number) {
this.pendingGoodServer.add(serverId)
}
getGoodFollowingServerIds () {
return Array.from(this.pendingGoodServer)
}
clearGoodFollowingServerIds () {
this.pendingGoodServer = new Set<number>()
}
getPendingFollowsScore () {
return this.pendingFollowsScore
}

2
server/lib/job-queue/handlers/activitypub-http-fetcher.ts
View File

@ -37,7 +37,7 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
'video-likes': items => createRates(items, video, 'like'),
'video-dislikes': items => createRates(items, video, 'dislike'),
'video-shares': items => addVideoShares(items, video),
'video-comments': items => addVideoComments(items, video),
'video-comments': items => addVideoComments(items),
'account-playlists': items => createAccountPlaylists(items, account)
}

13
server/lib/schedulers/actor-follow-scheduler.ts
View File

@ -2,7 +2,7 @@ import { isTestInstance } from '../../helpers/core-utils'
import { logger } from '../../helpers/logger'
import { ActorFollowModel } from '../../models/activitypub/actor-follow'
import { AbstractScheduler } from './abstract-scheduler'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { ActorFollowScoreCache } from '../files-cache'
export class ActorFollowScheduler extends AbstractScheduler {
@ -22,13 +22,20 @@ export class ActorFollowScheduler extends AbstractScheduler {
}
private async processPendingScores () {
const pendingScores = ActorFollowScoreCache.Instance.getPendingFollowsScoreCopy()
const pendingScores = ActorFollowScoreCache.Instance.getPendingFollowsScore()
const badServerIds = ActorFollowScoreCache.Instance.getBadFollowingServerIds()
const goodServerIds = ActorFollowScoreCache.Instance.getGoodFollowingServerIds()
ActorFollowScoreCache.Instance.clearPendingFollowsScore()
ActorFollowScoreCache.Instance.clearBadFollowingServerIds()
ActorFollowScoreCache.Instance.clearGoodFollowingServerIds()
for (const inbox of Object.keys(pendingScores)) {
await ActorFollowModel.updateFollowScore(inbox, pendingScores[inbox])
await ActorFollowModel.updateScore(inbox, pendingScores[inbox])
}
await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY)
await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS)
}
private async removeBadActorFollows () {

24
server/models/account/account-video-rate.ts
View File

@ -6,7 +6,7 @@ import { CONSTRAINTS_FIELDS, VIDEO_RATE_TYPES } from '../../initializers/constan
import { VideoModel } from '../video/video'
import { AccountModel } from './account'
import { ActorModel } from '../activitypub/actor'
import { getSort, throwIfNotValid } from '../utils'
import { buildLocalAccountIdsIn, getSort, throwIfNotValid } from '../utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { AccountVideoRate } from '../../../shared'
import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from '../video/video-channel'
@ -219,25 +219,11 @@ export class AccountVideoRateModel extends Model<AccountVideoRateModel> {
[Op.lt]: beforeUpdatedAt
},
videoId,
type
},
include: [
{
model: AccountModel.unscoped(),
required: true,
include: [
{
model: ActorModel.unscoped(),
required: true,
where: {
serverId: {
[Op.ne]: null
}
}
}
]
type,
accountId: {
[Op.notIn]: buildLocalAccountIdsIn()
}
],
},
transaction: t
}

26
server/models/activitypub/actor-follow.ts
View File

@ -23,7 +23,7 @@ import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES } from '../../initializers/constants'
import { ServerModel } from '../server/server'
import { getSort } from '../utils'
import { createSafeIn, getSort } from '../utils'
import { ActorModel, unusedActorAttributesForAPI } from './actor'
import { VideoChannelModel } from '../video/video-channel'
import { AccountModel } from '../account/account'
@ -464,7 +464,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
}
}
static updateFollowScore (inboxUrl: string, value: number, t?: Transaction) {
static updateScore (inboxUrl: string, value: number, t?: Transaction) {
const query = `UPDATE "actorFollow" SET "score" = LEAST("score" + ${value}, ${ACTOR_FOLLOW_SCORE.MAX}) ` +
'WHERE id IN (' +
'SELECT "actorFollow"."id" FROM "actorFollow" ' +
@ -480,6 +480,28 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
return ActorFollowModel.sequelize.query(query, options)
}
static async updateScoreByFollowingServers (serverIds: number[], value: number, t?: Transaction) {
if (serverIds.length === 0) return
const me = await getServerActor()
const serverIdsString = createSafeIn(ActorFollowModel, serverIds)
const query = `UPDATE "actorFollow" SET "score" = "score" + ${value} ` +
'WHERE id IN (' +
'SELECT "actorFollow"."id" FROM "actorFollow" ' +
'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."targetActorId" ' +
`WHERE "actorFollow"."actorId" = ${me.Account.actorId} ` + // I'm the follower
`AND "actor"."serverId" IN (${serverIdsString})` + // Criteria on followings
')'
const options = {
type: QueryTypes.BULKUPDATE,
transaction: t
}
return ActorFollowModel.sequelize.query(query, options)
}
private static async createListAcceptedFollowForApiQuery (
type: 'followers' | 'following',
actorIds: number[],

20
server/models/utils.ts
View File

@ -1,7 +1,7 @@
import { Model, Sequelize } from 'sequelize-typescript'
import * as validator from 'validator'
import { Col } from 'sequelize/types/lib/utils'
import { OrderItem } from 'sequelize/types'
import { OrderItem, literal } from 'sequelize'
type SortType = { sortModel: any, sortValue: string }
@ -129,16 +129,30 @@ function parseAggregateResult (result: any) {
return total
}
const createSafeIn = (model: typeof Model, stringArr: string[]) => {
return stringArr.map(t => model.sequelize.escape(t))
const createSafeIn = (model: typeof Model, stringArr: (string | number)[]) => {
return stringArr.map(t => model.sequelize.escape('' + t))
.join(', ')
}
function buildLocalAccountIdsIn () {
return literal(
'(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)'
)
}
function buildLocalActorIdsIn () {
return literal(
'(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)'
)
}
// ---------------------------------------------------------------------------
export {
buildBlockedAccountSQL,
buildLocalActorIdsIn,
SortType,
buildLocalAccountIdsIn,
getSort,
getVideoSort,
getSortOnModel,

38
server/models/video/video-comment.ts
View File

@ -22,7 +22,7 @@ import { AccountModel } from '../account/account'
import { ActorModel } from '../activitypub/actor'
import { AvatarModel } from '../avatar/avatar'
import { ServerModel } from '../server/server'
import { buildBlockedAccountSQL, getSort, throwIfNotValid } from '../utils'
import { buildBlockedAccountSQL, buildLocalAccountIdsIn, getSort, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { VideoChannelModel } from './video-channel'
import { getServerActor } from '../../helpers/utils'
@ -30,7 +30,7 @@ import { UserModel } from '../account/user'
import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor'
import { regexpCapture } from '../../helpers/regexp'
import { uniq } from 'lodash'
import { FindOptions, Op, Order, ScopeOptions, Sequelize, Transaction } from 'sequelize'
import { FindOptions, literal, Op, Order, ScopeOptions, Sequelize, Transaction } from 'sequelize'
enum ScopeNames {
WITH_ACCOUNT = 'WITH_ACCOUNT',
@ -281,16 +281,22 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query)
}
static loadByUrlAndPopulateReplyAndVideo (url: string, t?: Transaction) {
static loadByUrlAndPopulateReplyAndVideoUrlAndAccount (url: string, t?: Transaction) {
const query: FindOptions = {
where: {
url
}
},
include: [
{
attributes: [ 'id', 'url' ],
model: VideoModel.unscoped()
}
]
}
if (t !== undefined) query.transaction = t
return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_VIDEO ]).findOne(query)
return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_ACCOUNT ]).findOne(query)
}
static async listThreadsForApi (parameters: {
@ -471,25 +477,11 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
updatedAt: {
[Op.lt]: beforeUpdatedAt
},
videoId
},
include: [
{
required: true,
model: AccountModel.unscoped(),
include: [
{
required: true,
model: ActorModel.unscoped(),
where: {
serverId: {
[Op.ne]: null
}
}
}
]
videoId,
accountId: {
[Op.notIn]: buildLocalAccountIdsIn()
}
]
}
}
return VideoCommentModel.destroy(query)

18
server/models/video/video-share.ts
View File

@ -4,7 +4,7 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
import { AccountModel } from '../account/account'
import { ActorModel } from '../activitypub/actor'
import { throwIfNotValid } from '../utils'
import { buildLocalActorIdsIn, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { VideoChannelModel } from './video-channel'
import { Op, Transaction } from 'sequelize'
@ -207,19 +207,11 @@ export class VideoShareModel extends Model<VideoShareModel> {
updatedAt: {
[Op.lt]: beforeUpdatedAt
},
videoId
},
include: [
{
model: ActorModel.unscoped(),
required: true,
where: {
serverId: {
[ Op.ne ]: null
}
}
videoId,
actorId: {
[Op.notIn]: buildLocalActorIdsIn()
}
]
}
}
return VideoShareModel.destroy(query)

54
server/tests/api/server/handle-down.ts
View File

@ -19,8 +19,9 @@ import {
setAccessTokensToServers,
unfollow,
updateVideo,
uploadVideo,
wait
uploadVideo, uploadVideoAndGetId,
wait,
setActorFollowScores, closeAllSequelize
} from '../../../../shared/extra-utils'
import { follow, getFollowersListPaginationAndSort } from '../../../../shared/extra-utils/server/follows'
import { getJobsListPaginationAndSort, waitJobs } from '../../../../shared/extra-utils/server/jobs'
@ -43,6 +44,8 @@ describe('Test handle downs', function () {
let missedVideo2: Video
let unlistedVideo: Video
let videoIdsServer1: number[] = []
const videoAttributes = {
name: 'my super name for server 1',
category: 5,
@ -299,7 +302,54 @@ describe('Test handle downs', function () {
}
})
it('Should upload many videos on server 1', async function () {
this.timeout(120000)
for (let i = 0; i < 10; i++) {
const uuid = (await uploadVideoAndGetId({ server: servers[ 0 ], videoName: 'video ' + i })).uuid
videoIdsServer1.push(uuid)
}
await waitJobs(servers)
for (const id of videoIdsServer1) {
await getVideo(servers[ 1 ].url, id)
}
await waitJobs(servers)
await setActorFollowScores(servers[1].internalServerNumber, 20)
// Wait video expiration
await wait(11000)
// Refresh video -> score + 10 = 30
await getVideo(servers[1].url, videoIdsServer1[0])
await waitJobs(servers)
})
it('Should remove followings that are down', async function () {
this.timeout(120000)
killallServers([ servers[0] ])
// Wait video expiration
await wait(11000)
for (let i = 0; i < 3; i++) {
await getVideo(servers[1].url, videoIdsServer1[i])
await wait(1000)
await waitJobs([ servers[1] ])
}
for (const id of videoIdsServer1) {
await getVideo(servers[1].url, id, 403)
}
})
after(async function () {
await closeAllSequelize([ servers[1] ])
await cleanupTests(servers)
})
})

9
shared/extra-utils/miscs/sql.ts
View File

@ -80,11 +80,20 @@ function setPluginVersion (internalServerNumber: number, pluginName: string, new
return seq.query(`UPDATE "plugin" SET "version" = '${newVersion}' WHERE "name" = '${pluginName}'`, options)
}
function setActorFollowScores (internalServerNumber: number, newScore: number) {
const seq = getSequelize(internalServerNumber)
const options = { type: QueryTypes.UPDATE }
return seq.query(`UPDATE "actorFollow" SET "score" = ${newScore}`, options)
}
export {
setVideoField,
setPlaylistField,
setActorField,
countVideoViewsOf,
setPluginVersion,
setActorFollowScores,
closeAllSequelize
}