From 7ccddd7b5250bd25a917a6e77e58b87b9484a2a4 Mon Sep 17 00:00:00 2001 From: Josh Morel Date: Tue, 2 Apr 2019 05:26:47 -0400 Subject: [PATCH] add quarantine videos feature (#1637) * add quarantine videos feature * increase Notification settings test timeout to 20000ms. was completing 7000 locally but timing out after 10000 on travis * fix quarantine video test issues -propagate misspelling -remove skip from server/tests/client.ts * WIP use blacklist for moderator video approval instead of video.quarantine boolean * finish auto-blacklist feature --- client/src/app/+admin/admin.module.ts | 8 +- .../edit-custom-config.component.html | 17 ++ .../edit-custom-config.component.ts | 7 + client/src/app/+admin/moderation/index.ts | 1 + .../moderation/moderation.component.html | 4 +- .../+admin/moderation/moderation.component.ts | 11 +- .../+admin/moderation/moderation.routes.ts | 17 ++ .../video-auto-blacklist-list/index.ts | 1 + .../video-auto-blacklist-list.component.html | 49 +++++ .../video-auto-blacklist-list.component.scss | 94 +++++++++ .../video-auto-blacklist-list.component.ts | 100 +++++++++ .../video-blacklist-list.component.ts | 13 +- ...ount-notification-preferences.component.ts | 3 + .../my-account-videos.component.scss | 1 + client/src/app/core/server/server.service.ts | 7 + .../shared/users/user-notification.model.ts | 6 + .../users/user-notifications.component.html | 8 + .../video-blacklist.service.ts | 51 ++++- config/default.yaml | 6 + config/production.yaml.example | 6 + server/controllers/api/config.ts | 14 ++ .../controllers/api/users/my-notifications.ts | 1 + server/controllers/api/videos/blacklist.ts | 25 ++- server/controllers/api/videos/import.ts | 10 +- server/controllers/api/videos/index.ts | 18 +- .../custom-validators/video-blacklist.ts | 7 + server/helpers/video.ts | 3 + server/initializers/checker-before-init.ts | 2 +- server/initializers/constants.ts | 9 +- .../migrations/0350-video-blacklist-type.ts | 64 ++++++ server/lib/activitypub/videos.ts | 2 +- server/lib/emailer.ts | 23 +++ server/lib/job-queue/handlers/video-import.ts | 7 +- .../job-queue/handlers/video-transcoding.ts | 12 +- server/lib/notifier.ts | 63 +++++- .../lib/schedulers/update-videos-scheduler.ts | 2 +- server/lib/user.ts | 1 + server/lib/video-blacklist.ts | 31 +++ .../validators/videos/video-blacklist.ts | 25 ++- .../account/user-notification-setting.ts | 10 + server/models/video/schedule-video-update.ts | 3 +- server/models/video/video-blacklist.ts | 56 +++-- server/tests/api/check-params/config.ts | 7 + .../api/check-params/user-notifications.ts | 1 + .../tests/api/check-params/video-blacklist.ts | 11 +- server/tests/api/check-params/videos.ts | 3 +- server/tests/api/server/config.ts | 9 + server/tests/api/users/user-notifications.ts | 191 +++++++++++++++++- server/tests/api/videos/video-blacklist.ts | 25 ++- shared/models/server/custom-config.model.ts | 9 + shared/models/server/server-config.model.ts | 8 + .../users/user-notification-setting.model.ts | 1 + .../models/users/user-notification.model.ts | 4 +- .../videos/blacklist/video-blacklist.model.ts | 20 +- shared/utils/server/config.ts | 7 + shared/utils/users/user-notifications.ts | 35 +++- shared/utils/videos/video-blacklist.ts | 13 ++ shared/utils/videos/video-change-ownership.ts | 4 +- 58 files changed, 1047 insertions(+), 99 deletions(-) create mode 100644 client/src/app/+admin/moderation/video-auto-blacklist-list/index.ts create mode 100644 client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.html create mode 100644 client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.scss create mode 100644 client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.ts create mode 100644 server/initializers/migrations/0350-video-blacklist-type.ts create mode 100644 server/lib/video-blacklist.ts diff --git a/client/src/app/+admin/admin.module.ts b/client/src/app/+admin/admin.module.ts index f7f347105..282d59634 100644 --- a/client/src/app/+admin/admin.module.ts +++ b/client/src/app/+admin/admin.module.ts @@ -11,7 +11,12 @@ import { JobsComponent } from './jobs/job.component' import { JobsListComponent } from './jobs/jobs-list/jobs-list.component' import { JobService } from './jobs/shared/job.service' import { UserCreateComponent, UserListComponent, UsersComponent, UserUpdateComponent, UserPasswordComponent } from './users' -import { ModerationCommentModalComponent, VideoAbuseListComponent, VideoBlacklistListComponent } from './moderation' +import { + ModerationCommentModalComponent, + VideoAbuseListComponent, + VideoBlacklistListComponent, + VideoAutoBlacklistListComponent +} from './moderation' import { ModerationComponent } from '@app/+admin/moderation/moderation.component' import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' @@ -42,6 +47,7 @@ import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } f ModerationComponent, VideoBlacklistListComponent, VideoAbuseListComponent, + VideoAutoBlacklistListComponent, ModerationCommentModalComponent, InstanceServerBlocklistComponent, InstanceAccountBlocklistComponent, diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html index 6b654c67d..00a0d98f8 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html @@ -161,6 +161,23 @@ +
Auto-blacklist
+ + + + + +
+ +
+ + + + +
Administrator
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 45605e0fe..d8eb55da7 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -117,6 +117,13 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { threads: this.customConfigValidatorsService.TRANSCODING_THREADS, allowAdditionalExtensions: null, resolutions: {} + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: null + } + } } } diff --git a/client/src/app/+admin/moderation/index.ts b/client/src/app/+admin/moderation/index.ts index 66e2c6a39..3c683a28c 100644 --- a/client/src/app/+admin/moderation/index.ts +++ b/client/src/app/+admin/moderation/index.ts @@ -1,4 +1,5 @@ export * from './video-abuse-list' +export * from './video-auto-blacklist-list' export * from './video-blacklist-list' export * from './moderation.component' export * from './moderation.routes' diff --git a/client/src/app/+admin/moderation/moderation.component.html b/client/src/app/+admin/moderation/moderation.component.html index 01457936c..b70027957 100644 --- a/client/src/app/+admin/moderation/moderation.component.html +++ b/client/src/app/+admin/moderation/moderation.component.html @@ -4,7 +4,9 @@
Video abuses - Blacklisted videos + {{ autoBlacklistVideosEnabled ? 'Manually blacklisted videos' : 'Blacklisted videos' }} + + Auto-blacklisted videos Muted accounts diff --git a/client/src/app/+admin/moderation/moderation.component.ts b/client/src/app/+admin/moderation/moderation.component.ts index 2b2618933..47154af3f 100644 --- a/client/src/app/+admin/moderation/moderation.component.ts +++ b/client/src/app/+admin/moderation/moderation.component.ts @@ -1,13 +1,20 @@ import { Component } from '@angular/core' import { UserRight } from '../../../../../shared' -import { AuthService } from '@app/core/auth/auth.service' +import { AuthService, ServerService } from '@app/core' @Component({ templateUrl: './moderation.component.html', styleUrls: [ './moderation.component.scss' ] }) export class ModerationComponent { - constructor (private auth: AuthService) {} + autoBlacklistVideosEnabled: boolean + + constructor ( + private auth: AuthService, + private serverService: ServerService + ) { + this.autoBlacklistVideosEnabled = this.serverService.getConfig().autoBlacklist.videos.ofUsers.enabled + } hasVideoAbusesRight () { return this.auth.getUser().hasRight(UserRight.MANAGE_VIDEO_ABUSES) diff --git a/client/src/app/+admin/moderation/moderation.routes.ts b/client/src/app/+admin/moderation/moderation.routes.ts index 6f6dde290..a024f2bee 100644 --- a/client/src/app/+admin/moderation/moderation.routes.ts +++ b/client/src/app/+admin/moderation/moderation.routes.ts @@ -3,6 +3,7 @@ import { UserRight } from '../../../../../shared' import { UserRightGuard } from '@app/core' import { VideoAbuseListComponent } from '@app/+admin/moderation/video-abuse-list' import { VideoBlacklistListComponent } from '@app/+admin/moderation/video-blacklist-list' +import { VideoAutoBlacklistListComponent } from '@app/+admin/moderation/video-auto-blacklist-list' import { ModerationComponent } from '@app/+admin/moderation/moderation.component' import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' @@ -26,6 +27,11 @@ export const ModerationRoutes: Routes = [ redirectTo: 'video-blacklist/list', pathMatch: 'full' }, + { + path: 'video-auto-blacklist', + redirectTo: 'video-auto-blacklist/list', + pathMatch: 'full' + }, { path: 'video-abuses/list', component: VideoAbuseListComponent, @@ -37,6 +43,17 @@ export const ModerationRoutes: Routes = [ } } }, + { + path: 'video-auto-blacklist/list', + component: VideoAutoBlacklistListComponent, + canActivate: [ UserRightGuard ], + data: { + userRight: UserRight.MANAGE_VIDEO_BLACKLIST, + meta: { + title: 'Auto-blacklisted videos' + } + } + }, { path: 'video-blacklist/list', component: VideoBlacklistListComponent, diff --git a/client/src/app/+admin/moderation/video-auto-blacklist-list/index.ts b/client/src/app/+admin/moderation/video-auto-blacklist-list/index.ts new file mode 100644 index 000000000..e3522f68c --- /dev/null +++ b/client/src/app/+admin/moderation/video-auto-blacklist-list/index.ts @@ -0,0 +1 @@ +export * from './video-auto-blacklist-list.component' diff --git a/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.html b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.html new file mode 100644 index 000000000..fe579ffd7 --- /dev/null +++ b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.html @@ -0,0 +1,49 @@ +
No results.
+
+
+
+
+ +
+ + +
+ {{ video.name }} +
{{ video.account.displayName }}
+
{{ video.publishedAt | myFromNow }}
+
Privacy: {{ video.privacy.label }}
+
Sensitve: {{ video.nsfw }}
+
+ + +
+
+ + Cancel + + + + + Unblacklist + +
+
+ +
+ +
+
+ +
\ No newline at end of file diff --git a/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.scss b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.scss new file mode 100644 index 000000000..a73c17eb9 --- /dev/null +++ b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.scss @@ -0,0 +1,94 @@ +@import '_variables'; +@import '_mixins'; + +.action-selection-mode { + width: 194px; + display: flex; + justify-content: flex-end; + + .action-selection-mode-child { + position: fixed; + + .action-button { + display: inline-block; + } + + .action-button-cancel-selection { + @include peertube-button; + @include grey-button; + + margin-right: 10px; + } + + .action-button-unblacklist-selection { + @include peertube-button; + @include orange-button; + @include button-with-icon(21px); + + my-global-icon { + @include apply-svg-color(#fff); + } + } + } +} + +.video { + @include row-blocks; + + &:first-child { + margin-top: 47px; + } + + .checkbox-container { + display: flex; + align-items: center; + margin-right: 20px; + margin-left: 12px; + } + + my-video-thumbnail { + margin-right: 10px; + } + + .video-info { + flex-grow: 1; + + .video-info-name { + @include disable-default-a-behaviour; + + color: var(--mainForegroundColor); + display: block; + width: fit-content; + font-size: 16px; + font-weight: $font-semibold; + } + } + + .video-buttons { + min-width: 190px; + } +} + +@media screen and (max-width: $small-view) { + .video { + flex-direction: column; + height: auto; + text-align: center; + + .video-info-name { + margin: auto; + } + + input[type=checkbox] { + display: none; + } + + my-video-thumbnail { + margin-right: 0; + } + + .video-buttons { + margin-top: 10px; + } + } +} diff --git a/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.ts b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.ts new file mode 100644 index 000000000..b79f574c9 --- /dev/null +++ b/client/src/app/+admin/moderation/video-auto-blacklist-list/video-auto-blacklist-list.component.ts @@ -0,0 +1,100 @@ +import { Component, OnInit, OnDestroy } from '@angular/core' +import { Location } from '@angular/common' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { Router, ActivatedRoute } from '@angular/router' +import { AbstractVideoList } from '@app/shared/video/abstract-video-list' +import { ComponentPagination } from '@app/shared/rest/component-pagination.model' +import { Notifier, AuthService } from '@app/core' +import { Video } from '@shared/models' +import { VideoBlacklistService } from '@app/shared' +import { immutableAssign } from '@app/shared/misc/utils' +import { ScreenService } from '@app/shared/misc/screen.service' + +@Component({ + selector: 'my-video-auto-blacklist-list', + templateUrl: './video-auto-blacklist-list.component.html', + styleUrls: [ './video-auto-blacklist-list.component.scss' ] +}) +export class VideoAutoBlacklistListComponent extends AbstractVideoList implements OnInit, OnDestroy { + titlePage: string + currentRoute = '/admin/moderation/video-auto-blacklist/list' + checkedVideos: { [ id: number ]: boolean } = {} + pagination: ComponentPagination = { + currentPage: 1, + itemsPerPage: 5, + totalItems: null + } + + protected baseVideoWidth = -1 + protected baseVideoHeight = 155 + + constructor ( + protected router: Router, + protected route: ActivatedRoute, + protected i18n: I18n, + protected notifier: Notifier, + protected location: Location, + protected authService: AuthService, + protected screenService: ScreenService, + private videoBlacklistService: VideoBlacklistService, + ) { + super() + + this.titlePage = this.i18n('Auto-blacklisted videos') + } + + ngOnInit () { + super.ngOnInit() + } + + ngOnDestroy () { + super.ngOnDestroy() + } + + abortSelectionMode () { + this.checkedVideos = {} + } + + isInSelectionMode () { + return Object.keys(this.checkedVideos).some(k => this.checkedVideos[k] === true) + } + + getVideosObservable (page: number) { + const newPagination = immutableAssign(this.pagination, { currentPage: page }) + + return this.videoBlacklistService.getAutoBlacklistedAsVideoList(newPagination) + } + + generateSyndicationList () { + throw new Error('Method not implemented.') + } + + removeVideoFromBlacklist (entry: Video) { + this.videoBlacklistService.removeVideoFromBlacklist(entry.id).subscribe( + () => { + this.notifier.success(this.i18n('Video {{name}} removed from blacklist.', { name: entry.name })) + this.reloadVideos() + }, + + error => this.notifier.error(error.message) + ) + } + + removeSelectedVideosFromBlacklist () { + const toReleaseVideosIds = Object.keys(this.checkedVideos) + .filter(k => this.checkedVideos[ k ] === true) + .map(k => parseInt(k, 10)) + + this.videoBlacklistService.removeVideoFromBlacklist(toReleaseVideosIds).subscribe( + () => { + this.notifier.success(this.i18n('{{num}} videos removed from blacklist.', { num: toReleaseVideosIds.length })) + + this.abortSelectionMode() + this.reloadVideos() + }, + + error => this.notifier.error(error.message) + ) + } + +} diff --git a/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.ts b/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.ts index 5443d816d..f4bce7c48 100644 --- a/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.ts +++ b/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.ts @@ -1,9 +1,9 @@ import { Component, OnInit } from '@angular/core' import { SortMeta } from 'primeng/components/common/sortmeta' -import { Notifier } from '@app/core' +import { Notifier, ServerService } from '@app/core' import { ConfirmService } from '../../../core' import { RestPagination, RestTable, VideoBlacklistService } from '../../../shared' -import { VideoBlacklist } from '../../../../../../shared' +import { VideoBlacklist, VideoBlacklistType } from '../../../../../../shared' import { I18n } from '@ngx-translate/i18n-polyfill' import { DropdownAction } from '../../../shared/buttons/action-dropdown.component' import { Video } from '../../../shared/video/video.model' @@ -20,11 +20,13 @@ export class VideoBlacklistListComponent extends RestTable implements OnInit { rowsPerPage = 10 sort: SortMeta = { field: 'createdAt', order: 1 } pagination: RestPagination = { count: this.rowsPerPage, start: 0 } + listBlacklistTypeFilter: VideoBlacklistType = undefined videoBlacklistActions: DropdownAction[] = [] constructor ( private notifier: Notifier, + private serverService: ServerService, private confirmService: ConfirmService, private videoBlacklistService: VideoBlacklistService, private markdownRenderer: MarkdownService, @@ -32,6 +34,11 @@ export class VideoBlacklistListComponent extends RestTable implements OnInit { ) { super() + // don't filter if auto-blacklist not enabled as this will be only list + if (this.serverService.getConfig().autoBlacklist.videos.ofUsers.enabled) { + this.listBlacklistTypeFilter = VideoBlacklistType.MANUAL + } + this.videoBlacklistActions = [ { label: this.i18n('Unblacklist'), @@ -77,7 +84,7 @@ export class VideoBlacklistListComponent extends RestTable implements OnInit { } protected loadData () { - this.videoBlacklistService.listBlacklist(this.pagination, this.sort) + this.videoBlacklistService.listBlacklist(this.pagination, this.sort, this.listBlacklistTypeFilter) .subscribe( async resultList => { this.totalRecords = resultList.total diff --git a/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts b/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts index 8d4f2c837..67ddf54da 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts @@ -31,10 +31,12 @@ export class MyAccountNotificationPreferencesComponent implements OnInit { private serverService: ServerService, private notifier: Notifier ) { + this.labelNotifications = { newVideoFromSubscription: this.i18n('New video from your subscriptions'), newCommentOnMyVideo: this.i18n('New comment on your video'), videoAbuseAsModerator: this.i18n('New video abuse'), + videoAutoBlacklistAsModerator: this.i18n('Video auto-blacklisted waiting review'), blacklistOnMyVideo: this.i18n('One of your video is blacklisted/unblacklisted'), myVideoPublished: this.i18n('Video published (after transcoding/scheduled update)'), myVideoImportFinished: this.i18n('Video import finished'), @@ -46,6 +48,7 @@ export class MyAccountNotificationPreferencesComponent implements OnInit { this.rightNotifications = { videoAbuseAsModerator: UserRight.MANAGE_VIDEO_ABUSES, + videoAutoBlacklistAsModerator: UserRight.MANAGE_VIDEO_BLACKLIST, newUserRegistration: UserRight.MANAGE_USERS } diff --git a/client/src/app/+my-account/my-account-videos/my-account-videos.component.scss b/client/src/app/+my-account/my-account-videos/my-account-videos.component.scss index f6b5faa45..d2df6f290 100644 --- a/client/src/app/+my-account/my-account-videos/my-account-videos.component.scss +++ b/client/src/app/+my-account/my-account-videos/my-account-videos.component.scss @@ -82,6 +82,7 @@ } } } + } } diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index acaca8a01..b0c5d1130 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -98,6 +98,13 @@ export class ServerService { videos: { intervalDays: 0 } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } } } private videoCategories: Array> = [] diff --git a/client/src/app/shared/users/user-notification.model.ts b/client/src/app/shared/users/user-notification.model.ts index 097830752..7d0eb5ea2 100644 --- a/client/src/app/shared/users/user-notification.model.ts +++ b/client/src/app/shared/users/user-notification.model.ts @@ -54,6 +54,7 @@ export class UserNotification implements UserNotificationServer { videoUrl?: string commentUrl?: any[] videoAbuseUrl?: string + videoAutoBlacklistUrl?: string accountUrl?: string videoImportIdentifier?: string videoImportUrl?: string @@ -107,6 +108,11 @@ export class UserNotification implements UserNotificationServer { this.videoUrl = this.buildVideoUrl(this.videoAbuse.video) break + case UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS: + this.videoAutoBlacklistUrl = '/admin/moderation/video-auto-blacklist/list' + this.videoUrl = this.buildVideoUrl(this.video) + break + case UserNotificationType.BLACKLIST_ON_MY_VIDEO: this.videoUrl = this.buildVideoUrl(this.videoBlacklist.video) break diff --git a/client/src/app/shared/users/user-notifications.component.html b/client/src/app/shared/users/user-notifications.component.html index 1c0af1bb0..6d2f2750e 100644 --- a/client/src/app/shared/users/user-notifications.component.html +++ b/client/src/app/shared/users/user-notifications.component.html @@ -36,6 +36,14 @@
+ + + +
+ The recently added video {{ notification.video.name }} has been auto-blacklisted +
+ + diff --git a/client/src/app/shared/video-blacklist/video-blacklist.service.ts b/client/src/app/shared/video-blacklist/video-blacklist.service.ts index 94e46d7c2..a9eab9b6f 100644 --- a/client/src/app/shared/video-blacklist/video-blacklist.service.ts +++ b/client/src/app/shared/video-blacklist/video-blacklist.service.ts @@ -1,11 +1,13 @@ -import { catchError, map } from 'rxjs/operators' +import { catchError, map, concatMap, toArray } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { SortMeta } from 'primeng/components/common/sortmeta' -import { Observable } from 'rxjs' -import { VideoBlacklist, ResultList } from '../../../../../shared' +import { from as observableFrom, Observable } from 'rxjs' +import { VideoBlacklist, VideoBlacklistType, ResultList } from '../../../../../shared' +import { Video } from '../video/video.model' import { environment } from '../../../environments/environment' import { RestExtractor, RestPagination, RestService } from '../rest' +import { ComponentPagination } from '../rest/component-pagination.model' @Injectable() export class VideoBlacklistService { @@ -17,10 +19,14 @@ export class VideoBlacklistService { private restExtractor: RestExtractor ) {} - listBlacklist (pagination: RestPagination, sort: SortMeta): Observable> { + listBlacklist (pagination: RestPagination, sort: SortMeta, type?: VideoBlacklistType): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) + if (type) { + params = params.set('type', type.toString()) + } + return this.authHttp.get>(VideoBlacklistService.BASE_VIDEOS_URL + 'blacklist', { params }) .pipe( map(res => this.restExtractor.convertResultListDateToHuman(res)), @@ -28,12 +34,37 @@ export class VideoBlacklistService { ) } - removeVideoFromBlacklist (videoId: number) { - return this.authHttp.delete(VideoBlacklistService.BASE_VIDEOS_URL + videoId + '/blacklist') - .pipe( - map(this.restExtractor.extractDataBool), - catchError(res => this.restExtractor.handleError(res)) - ) + getAutoBlacklistedAsVideoList (videoPagination: ComponentPagination): Observable<{ videos: Video[], totalVideos: number}> { + const pagination = this.restService.componentPaginationToRestPagination(videoPagination) + + // prioritize first created since waiting longest + const AUTO_BLACKLIST_SORT = 'createdAt' + + let params = new HttpParams() + params = this.restService.addRestGetParams(params, pagination, AUTO_BLACKLIST_SORT) + + params = params.set('type', VideoBlacklistType.AUTO_BEFORE_PUBLISHED.toString()) + + return this.authHttp.get>(VideoBlacklistService.BASE_VIDEOS_URL + 'blacklist', { params }) + .pipe( + map(res => { + const videos = res.data.map(videoBlacklist => new Video(videoBlacklist.video)) + const totalVideos = res.total + return { videos, totalVideos } + }), + catchError(res => this.restExtractor.handleError(res)) + ) + } + + removeVideoFromBlacklist (videoIdArgs: number | number[]) { + const videoIds = Array.isArray(videoIdArgs) ? videoIdArgs : [ videoIdArgs ] + + return observableFrom(videoIds) + .pipe( + concatMap(id => this.authHttp.delete(VideoBlacklistService.BASE_VIDEOS_URL + id + '/blacklist')), + toArray(), + catchError(err => this.restExtractor.handleError(err)) + ) } blacklistVideo (videoId: number, reason: string, unfederate: boolean) { diff --git a/config/default.yaml b/config/default.yaml index c5bf8e457..615910478 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -162,6 +162,12 @@ import: torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) enabled: false +auto_blacklist: + # New videos automatically blacklisted so moderators can review before publishing + videos: + of_users: + enabled: false + instance: name: 'PeerTube' short_description: 'PeerTube, a federated (ActivityPub) video streaming platform using P2P (BitTorrent) directly in the web browser with WebTorrent and Angular.' diff --git a/config/production.yaml.example b/config/production.yaml.example index 306e5576d..5299484a5 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -176,6 +176,12 @@ import: torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) enabled: false +auto_blacklist: + # New videos automatically blacklisted so moderators can review before publishing + videos: + of_users: + enabled: false + # Instance settings instance: name: 'PeerTube' diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 6497cda3c..bd0ba4f9d 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -94,6 +94,13 @@ async function getConfig (req: express.Request, res: express.Response) { } } }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED + } + } + }, avatar: { file: { size: { @@ -265,6 +272,13 @@ function customConfig (): CustomConfig { enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED } } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED + } + } } } } diff --git a/server/controllers/api/users/my-notifications.ts b/server/controllers/api/users/my-notifications.ts index bbafda5a6..4edad2a74 100644 --- a/server/controllers/api/users/my-notifications.ts +++ b/server/controllers/api/users/my-notifications.ts @@ -69,6 +69,7 @@ async function updateNotificationSettings (req: express.Request, res: express.Re newVideoFromSubscription: body.newVideoFromSubscription, newCommentOnMyVideo: body.newCommentOnMyVideo, videoAbuseAsModerator: body.videoAbuseAsModerator, + videoAutoBlacklistAsModerator: body.videoAutoBlacklistAsModerator, blacklistOnMyVideo: body.blacklistOnMyVideo, myVideoPublished: body.myVideoPublished, myVideoImportFinished: body.myVideoImportFinished, diff --git a/server/controllers/api/videos/blacklist.ts b/server/controllers/api/videos/blacklist.ts index d0728eb59..27dcfb761 100644 --- a/server/controllers/api/videos/blacklist.ts +++ b/server/controllers/api/videos/blacklist.ts @@ -1,5 +1,5 @@ import * as express from 'express' -import { UserRight, VideoBlacklist, VideoBlacklistCreate } from '../../../../shared' +import { VideoBlacklist, UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../../../shared' import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' import { @@ -12,7 +12,8 @@ import { setDefaultPagination, videosBlacklistAddValidator, videosBlacklistRemoveValidator, - videosBlacklistUpdateValidator + videosBlacklistUpdateValidator, + videosBlacklistFiltersValidator } from '../../../middlewares' import { VideoBlacklistModel } from '../../../models/video/video-blacklist' import { sequelizeTypescript } from '../../../initializers' @@ -36,6 +37,7 @@ blacklistRouter.get('/blacklist', blacklistSortValidator, setBlacklistSort, setDefaultPagination, + videosBlacklistFiltersValidator, asyncMiddleware(listBlacklist) ) @@ -68,7 +70,8 @@ async function addVideoToBlacklist (req: express.Request, res: express.Response) const toCreate = { videoId: videoInstance.id, unfederated: body.unfederate === true, - reason: body.reason + reason: body.reason, + type: VideoBlacklistType.MANUAL } const blacklist = await VideoBlacklistModel.create(toCreate) @@ -98,7 +101,7 @@ async function updateVideoBlacklistController (req: express.Request, res: expres } async function listBlacklist (req: express.Request, res: express.Response, next: express.NextFunction) { - const resultList = await VideoBlacklistModel.listForApi(req.query.start, req.query.count, req.query.sort) + const resultList = await VideoBlacklistModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.type) return res.json(getFormattedObjects(resultList.data, resultList.total)) } @@ -107,18 +110,30 @@ async function removeVideoFromBlacklistController (req: express.Request, res: ex const videoBlacklist = res.locals.videoBlacklist const video = res.locals.video - await sequelizeTypescript.transaction(async t => { + const videoBlacklistType = await sequelizeTypescript.transaction(async t => { const unfederated = videoBlacklist.unfederated + const videoBlacklistType = videoBlacklist.type + await videoBlacklist.destroy({ transaction: t }) // Re federate the video if (unfederated === true) { await federateVideoIfNeeded(video, true, t) } + + return videoBlacklistType }) Notifier.Instance.notifyOnVideoUnblacklist(video) + if (videoBlacklistType === VideoBlacklistType.AUTO_BEFORE_PUBLISHED) { + Notifier.Instance.notifyOnVideoPublishedAfterRemovedFromAutoBlacklist(video) + + // Delete on object so new video notifications will send + delete video.VideoBlacklist + Notifier.Instance.notifyOnNewVideo(video) + } + logger.info('Video %s removed from blacklist.', res.locals.video.uuid) return res.type('json').status(204).end() diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index cbd2e8514..c234a1391 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -18,10 +18,12 @@ import { join } from 'path' import { isArray } from '../../../helpers/custom-validators/misc' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' import { VideoChannelModel } from '../../../models/video/video-channel' +import { UserModel } from '../../../models/account/user' import * as Bluebird from 'bluebird' import * as parseTorrent from 'parse-torrent' import { getSecureTorrentName } from '../../../helpers/utils' import { readFile, move } from 'fs-extra' +import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' const auditLogger = auditLoggerFactory('video-imports') const videoImportsRouter = express.Router() @@ -85,7 +87,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string } - const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }) + const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }, user) await processThumbnail(req, video) await processPreview(req, video) @@ -128,7 +130,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) }).end() } - const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) + const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo, user) const downloadThumbnail = !await processThumbnail(req, video) const downloadPreview = !await processPreview(req, video) @@ -156,7 +158,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) return res.json(videoImport.toFormattedJSON()).end() } -function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) { +function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo, user: UserModel) { const videoData = { name: body.name || importData.name || 'Unknown name', remote: false, @@ -218,6 +220,8 @@ function insertIntoDB ( const videoCreated = await video.save(sequelizeOptions) videoCreated.VideoChannel = videoChannel + await autoBlacklistVideoIfNeeded(video, videoChannel.Account.User, t) + // Set tags to the video if (tags) { const tagInstances = await TagModel.findOrCreateTags(tags, t) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 08bee97d3..393324819 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -6,6 +6,7 @@ import { processImage } from '../../../helpers/image-utils' import { logger } from '../../../helpers/logger' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { getFormattedObjects, getServerActor } from '../../../helpers/utils' +import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' import { CONFIG, MIMETYPES, @@ -193,6 +194,7 @@ async function addVideo (req: express.Request, res: express.Response) { channelId: res.locals.videoChannel.id, originallyPublishedAt: videoInfo.originallyPublishedAt } + const video = new VideoModel(videoData) video.url = getVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object @@ -237,7 +239,7 @@ async function addVideo (req: express.Request, res: express.Response) { // Create the torrent file await video.createTorrentAndSetInfoHash(videoFile) - const videoCreated = await sequelizeTypescript.transaction(async t => { + const { videoCreated, videoWasAutoBlacklisted } = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } const videoCreated = await video.save(sequelizeOptions) @@ -266,15 +268,23 @@ async function addVideo (req: express.Request, res: express.Response) { }, { transaction: t }) } - await federateVideoIfNeeded(video, true, t) + const videoWasAutoBlacklisted = await autoBlacklistVideoIfNeeded(video, res.locals.oauth.token.User, t) + + if (!videoWasAutoBlacklisted) { + await federateVideoIfNeeded(video, true, t) + } auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) - return videoCreated + return { videoCreated, videoWasAutoBlacklisted } }) - Notifier.Instance.notifyOnNewVideo(videoCreated) + if (videoWasAutoBlacklisted) { + Notifier.Instance.notifyOnVideoAutoBlacklist(videoCreated) + } else { + Notifier.Instance.notifyOnNewVideo(videoCreated) + } if (video.state === VideoState.TO_TRANSCODE) { // Put uuid because we don't have id auto incremented for now diff --git a/server/helpers/custom-validators/video-blacklist.ts b/server/helpers/custom-validators/video-blacklist.ts index 25f908228..465f58a9c 100644 --- a/server/helpers/custom-validators/video-blacklist.ts +++ b/server/helpers/custom-validators/video-blacklist.ts @@ -1,7 +1,9 @@ import { Response } from 'express' import * as validator from 'validator' +import { exists } from './misc' import { CONSTRAINTS_FIELDS } from '../../initializers' import { VideoBlacklistModel } from '../../models/video/video-blacklist' +import { VideoBlacklistType } from '../../../shared/models/videos' const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST @@ -24,9 +26,14 @@ async function doesVideoBlacklistExist (videoId: number, res: Response) { return true } +function isVideoBlacklistTypeValid (value: any) { + return exists(value) && validator.isInt('' + value) && VideoBlacklistType[value] !== undefined +} + // --------------------------------------------------------------------------- export { isVideoBlacklistReasonValid, + isVideoBlacklistTypeValid, doesVideoBlacklistExist } diff --git a/server/helpers/video.ts b/server/helpers/video.ts index c90fe06c7..f6f51a297 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts @@ -1,4 +1,7 @@ +import { CONFIG } from '../initializers' import { VideoModel } from '../models/video/video' +import { UserRight } from '../../shared' +import { UserModel } from '../models/account/user' type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index ef12b3eea..e26f38564 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -20,7 +20,7 @@ function checkMissedConfig () { 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'redundancy.videos.strategies', 'redundancy.videos.check_interval', 'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', - 'import.videos.http.enabled', 'import.videos.torrent.enabled', + 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index ff0ade17a..f59d3ef7a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -18,7 +18,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 345 +const LAST_MIGRATION_VERSION = 350 // --------------------------------------------------------------------------- @@ -288,6 +288,13 @@ const CONFIG = { } } }, + AUTO_BLACKLIST: { + VIDEOS: { + OF_USERS: { + get ENABLED () { return config.get('auto_blacklist.videos.of_users.enabled') } + } + } + }, CACHE: { PREVIEWS: { get SIZE () { return config.get('cache.previews.size') } diff --git a/server/initializers/migrations/0350-video-blacklist-type.ts b/server/initializers/migrations/0350-video-blacklist-type.ts new file mode 100644 index 000000000..4849020ef --- /dev/null +++ b/server/initializers/migrations/0350-video-blacklist-type.ts @@ -0,0 +1,64 @@ +import * as Sequelize from 'sequelize' +import { VideoBlacklistType } from '../../../shared/models/videos' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + const data = { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('videoBlacklist', 'type', data) + } + + { + const query = 'UPDATE "videoBlacklist" SET "type" = ' + VideoBlacklistType.MANUAL + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: null + } + await utils.queryInterface.changeColumn('videoBlacklist', 'type', data) + } + + { + const data = { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('userNotificationSetting', 'videoAutoBlacklistAsModerator', data) + } + + { + const query = 'UPDATE "userNotificationSetting" SET "videoAutoBlacklistAsModerator" = 3' + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: false + } + await utils.queryInterface.changeColumn('userNotificationSetting', 'videoAutoBlacklistAsModerator', data) + } +} +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 2c932371b..d935e3f90 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -45,7 +45,7 @@ import { VideoShareModel } from '../../models/video/video-share' import { VideoCommentModel } from '../../models/video/video-comment' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { - // If the video is not private and published, we federate it + // If the video is not private and is published, we federate it if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) { // Fetch more attributes that we will need to serialize in AP object if (isArray(video.VideoCaptions) === false) { diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 04e4b94b6..eec97c27e 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -250,6 +250,29 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + addVideoAutoBlacklistModeratorsNotification (to: string[], video: VideoModel) { + const VIDEO_AUTO_BLACKLIST_URL = CONFIG.WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' + const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() + + const text = `Hi,\n\n` + + `A recently added video was auto-blacklisted and requires moderator review before publishing.` + + `\n\n` + + `You can view it and take appropriate action on ${videoUrl}` + + `\n\n` + + `A full list of auto-blacklisted videos can be reviewed here: ${VIDEO_AUTO_BLACKLIST_URL}` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: '[PeerTube] An auto-blacklisted video is awaiting review', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + addNewUserRegistrationNotification (to: string[], user: UserModel) { const text = `Hi,\n\n` + `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` + diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index d96bfdf43..c5fc1061c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -196,9 +196,14 @@ async function processFile (downloader: () => Promise, videoImport: Vide return videoImportUpdated }) - Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) + if (videoImportUpdated.Video.VideoBlacklist) { + Notifier.Instance.notifyOnVideoAutoBlacklist(videoImportUpdated.Video) + } else { + Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) + } + // Create transcoding jobs? if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { // Put uuid because we don't have id auto incremented for now diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index d9dad795e..581ec283e 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -85,10 +85,9 @@ async function publishVideoIfNeeded (video: VideoModel, payload?: VideoTranscodi return { videoDatabase, videoPublished } }) - // don't notify prior to scheduled video update - if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { + if (videoPublished) { Notifier.Instance.notifyOnNewVideo(videoDatabase) - Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) } await createHlsJobIfEnabled(payload) @@ -146,11 +145,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: Video return { videoDatabase, videoPublished } }) - // don't notify prior to scheduled video update - if (!videoDatabase.ScheduleVideoUpdate) { - if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) - if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) - } + if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) } diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index 501680f6b..9fe93ec0d 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts @@ -23,19 +23,35 @@ class Notifier { private constructor () {} notifyOnNewVideo (video: VideoModel): void { - // Only notify on public and published videos - if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return + // Only notify on public and published videos which are not blacklisted + if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED || video.VideoBlacklist) return this.notifySubscribersOfNewVideo(video) .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) } - notifyOnPendingVideoPublished (video: VideoModel): void { - // Only notify on public videos that has been published while the user waited transcoding/scheduled update - if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return + notifyOnVideoPublishedAfterTranscoding (video: VideoModel): void { + // don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update + if (!video.waitTranscoding || video.VideoBlacklist || video.ScheduleVideoUpdate) return this.notifyOwnedVideoHasBeenPublished(video) - .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err })) + .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) + } + + notifyOnVideoPublishedAfterScheduledUpdate (video: VideoModel): void { + // don't notify if video is still blacklisted or waiting for transcoding + if (video.VideoBlacklist || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return + + this.notifyOwnedVideoHasBeenPublished(video) + .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) + } + + notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: VideoModel): void { + // don't notify if video is still waiting for transcoding or scheduled update + if (video.ScheduleVideoUpdate || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return + + this.notifyOwnedVideoHasBeenPublished(video) + .catch(err => logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err })) // tslint:disable-line:max-line-length } notifyOnNewComment (comment: VideoCommentModel): void { @@ -51,6 +67,11 @@ class Notifier { .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) } + notifyOnVideoAutoBlacklist (video: VideoModel): void { + this.notifyModeratorsOfVideoAutoBlacklist(video) + .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', video.url, { err })) + } + notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { this.notifyVideoOwnerOfBlacklist(videoBlacklist) .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) @@ -58,7 +79,7 @@ class Notifier { notifyOnVideoUnblacklist (video: VideoModel): void { this.notifyVideoOwnerOfUnblacklist(video) - .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) + .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) } notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { @@ -268,6 +289,34 @@ class Notifier { return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) } + private async notifyModeratorsOfVideoAutoBlacklist (video: VideoModel) { + const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) + if (moderators.length === 0) return + + logger.info('Notifying %s moderators of video auto-blacklist %s.', moderators.length, video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.videoAutoBlacklistAsModerator + } + async function notificationCreator (user: UserModel) { + + const notification = await UserNotificationModel.create({ + type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, + userId: user.id, + videoId: video.id + }) + notification.Video = video + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addVideoAutoBlacklistModeratorsNotification(emails, video) + } + + return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) + } + private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { const user = await UserModel.loadByVideoId(videoBlacklist.videoId) if (!user) return diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 2618a5857..2179a2f26 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -57,7 +57,7 @@ export class UpdateVideosScheduler extends AbstractScheduler { for (const v of publishedVideos) { Notifier.Instance.notifyOnNewVideo(v) - Notifier.Instance.notifyOnPendingVideoPublished(v) + Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v) } } diff --git a/server/lib/user.ts b/server/lib/user.ts index 02a84f15b..5588b0f76 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -106,6 +106,7 @@ function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Tr myVideoImportFinished: UserNotificationSettingValue.WEB, myVideoPublished: UserNotificationSettingValue.WEB, videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, newUserRegistration: UserNotificationSettingValue.WEB, commentMention: UserNotificationSettingValue.WEB, diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts new file mode 100644 index 000000000..dc4e0aed9 --- /dev/null +++ b/server/lib/video-blacklist.ts @@ -0,0 +1,31 @@ +import * as sequelize from 'sequelize' +import { CONFIG } from '../initializers/constants' +import { VideoBlacklistType, UserRight } from '../../shared/models' +import { VideoBlacklistModel } from '../models/video/video-blacklist' +import { UserModel } from '../models/account/user' +import { VideoModel } from '../models/video/video' +import { logger } from '../helpers/logger' + +async function autoBlacklistVideoIfNeeded (video: VideoModel, user: UserModel, transaction: sequelize.Transaction) { + if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED) return false + + if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) return false + + const sequelizeOptions = { transaction } + const videoBlacklistToCreate = { + videoId: video.id, + unfederated: true, + reason: 'Auto-blacklisted. Moderator review required.', + type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED + } + await VideoBlacklistModel.create(videoBlacklistToCreate, sequelizeOptions) + logger.info('Video %s auto-blacklisted.', video.uuid) + + return true +} + +// --------------------------------------------------------------------------- + +export { + autoBlacklistVideoIfNeeded +} diff --git a/server/middlewares/validators/videos/video-blacklist.ts b/server/middlewares/validators/videos/video-blacklist.ts index db318dcdb..1d7ddb2e3 100644 --- a/server/middlewares/validators/videos/video-blacklist.ts +++ b/server/middlewares/validators/videos/video-blacklist.ts @@ -1,10 +1,14 @@ import * as express from 'express' -import { body, param } from 'express-validator/check' +import { body, param, query } from 'express-validator/check' import { isBooleanValid, isIdOrUUIDValid } from '../../../helpers/custom-validators/misc' import { doesVideoExist } from '../../../helpers/custom-validators/videos' import { logger } from '../../../helpers/logger' import { areValidationErrors } from '../utils' -import { doesVideoBlacklistExist, isVideoBlacklistReasonValid } from '../../../helpers/custom-validators/video-blacklist' +import { + doesVideoBlacklistExist, + isVideoBlacklistReasonValid, + isVideoBlacklistTypeValid +} from '../../../helpers/custom-validators/video-blacklist' const videosBlacklistRemoveValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid videoId'), @@ -65,10 +69,25 @@ const videosBlacklistUpdateValidator = [ } ] +const videosBlacklistFiltersValidator = [ + query('type') + .optional() + .custom(isVideoBlacklistTypeValid).withMessage('Should have a valid video blacklist type attribute'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videos blacklist filters query', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + // --------------------------------------------------------------------------- export { videosBlacklistAddValidator, videosBlacklistRemoveValidator, - videosBlacklistUpdateValidator + videosBlacklistUpdateValidator, + videosBlacklistFiltersValidator } diff --git a/server/models/account/user-notification-setting.ts b/server/models/account/user-notification-setting.ts index f1c3ac223..ba7f739b9 100644 --- a/server/models/account/user-notification-setting.ts +++ b/server/models/account/user-notification-setting.ts @@ -56,6 +56,15 @@ export class UserNotificationSettingModel extends Model throwIfNotValid(value, isUserNotificationSettingValid, 'videoAutoBlacklistAsModerator') + ) + @Column + videoAutoBlacklistAsModerator: UserNotificationSettingValue + @AllowNull(false) @Default(null) @Is( @@ -139,6 +148,7 @@ export class UserNotificationSettingModel extends Model { model: VideoModel.scope( [ VideoScopeNames.WITH_FILES, - VideoScopeNames.WITH_ACCOUNT_DETAILS + VideoScopeNames.WITH_ACCOUNT_DETAILS, + VideoScopeNames.WITH_BLACKLISTED ] ) } diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts index 3b567e488..86b1f6acb 100644 --- a/server/models/video/video-blacklist.ts +++ b/server/models/video/video-blacklist.ts @@ -1,8 +1,21 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + Is, Model, + Table, + UpdatedAt, + IFindOptions +} from 'sequelize-typescript' import { getSortOnModel, SortType, throwIfNotValid } from '../utils' import { VideoModel } from './video' -import { isVideoBlacklistReasonValid } from '../../helpers/custom-validators/video-blacklist' -import { VideoBlacklist } from '../../../shared/models/videos' +import { VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel' +import { isVideoBlacklistReasonValid, isVideoBlacklistTypeValid } from '../../helpers/custom-validators/video-blacklist' +import { VideoBlacklist, VideoBlacklistType } from '../../../shared/models/videos' import { CONSTRAINTS_FIELDS } from '../../initializers' @Table({ @@ -25,6 +38,12 @@ export class VideoBlacklistModel extends Model { @Column unfederated: boolean + @AllowNull(false) + @Default(null) + @Is('VideoBlacklistType', value => throwIfNotValid(value, isVideoBlacklistTypeValid, 'type')) + @Column + type: VideoBlacklistType + @CreatedAt createdAt: Date @@ -43,19 +62,29 @@ export class VideoBlacklistModel extends Model { }) Video: VideoModel - static listForApi (start: number, count: number, sort: SortType) { - const query = { + static listForApi (start: number, count: number, sort: SortType, type?: VideoBlacklistType) { + const query: IFindOptions = { offset: start, limit: count, order: getSortOnModel(sort.sortModel, sort.sortValue), include: [ { model: VideoModel, - required: true + required: true, + include: [ + { + model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, true ] }), + required: true + } + ] } ] } + if (type) { + query.where = { type } + } + return VideoBlacklistModel.findAndCountAll(query) .then(({ rows, count }) => { return { @@ -76,26 +105,15 @@ export class VideoBlacklistModel extends Model { } toFormattedJSON (): VideoBlacklist { - const video = this.Video - return { id: this.id, createdAt: this.createdAt, updatedAt: this.updatedAt, reason: this.reason, unfederated: this.unfederated, + type: this.type, - video: { - id: video.id, - name: video.name, - uuid: video.uuid, - description: video.description, - duration: video.duration, - views: video.views, - likes: video.likes, - dislikes: video.dislikes, - nsfw: video.nsfw - } + video: this.Video.toFormattedJSON() } } } diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index c6b460f23..0b333e2f4 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -80,6 +80,13 @@ describe('Test config API validators', function () { enabled: false } } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } } } diff --git a/server/tests/api/check-params/user-notifications.ts b/server/tests/api/check-params/user-notifications.ts index 714f481e9..36eaceac7 100644 --- a/server/tests/api/check-params/user-notifications.ts +++ b/server/tests/api/check-params/user-notifications.ts @@ -168,6 +168,7 @@ describe('Test user notifications API validators', function () { newVideoFromSubscription: UserNotificationSettingValue.WEB, newCommentOnMyVideo: UserNotificationSettingValue.WEB, videoAbuseAsModerator: UserNotificationSettingValue.WEB, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB, blacklistOnMyVideo: UserNotificationSettingValue.WEB, myVideoImportFinished: UserNotificationSettingValue.WEB, myVideoPublished: UserNotificationSettingValue.WEB, diff --git a/server/tests/api/check-params/video-blacklist.ts b/server/tests/api/check-params/video-blacklist.ts index 6b82643f4..fc039e847 100644 --- a/server/tests/api/check-params/video-blacklist.ts +++ b/server/tests/api/check-params/video-blacklist.ts @@ -8,6 +8,7 @@ import { flushAndRunMultipleServers, flushTests, getBlacklistedVideosList, + getBlacklistedVideosListWithTypeFilter, getVideo, getVideoWithToken, killallServers, @@ -24,7 +25,7 @@ import { checkBadSortPagination, checkBadStartPagination } from '../../../../shared/utils/requests/check-api-params' -import { VideoDetails } from '../../../../shared/models/videos' +import { VideoDetails, VideoBlacklistType } from '../../../../shared/models/videos' import { expect } from 'chai' describe('Test video blacklist API validators', function () { @@ -238,6 +239,14 @@ describe('Test video blacklist API validators', function () { it('Should fail with an incorrect sort', async function () { await checkBadSortPagination(servers[0].url, basePath, servers[0].accessToken) }) + + it('Should fail with an invalid type', async function () { + await getBlacklistedVideosListWithTypeFilter(servers[0].url, servers[0].accessToken, 0, 400) + }) + + it('Should succeed with the correct parameters', async function () { + await getBlacklistedVideosListWithTypeFilter(servers[0].url, servers[0].accessToken, VideoBlacklistType.MANUAL) + }) }) after(async function () { diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 3eccaee44..5a013b890 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -7,7 +7,8 @@ import { join } from 'path' import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' import { createUser, flushTests, getMyUserInformation, getVideo, getVideosList, immutableAssign, killallServers, makeDeleteRequest, - makeGetRequest, makeUploadRequest, makePutBodyRequest, removeVideo, runServer, ServerInfo, setAccessTokensToServers, userLogin + makeGetRequest, makeUploadRequest, makePutBodyRequest, removeVideo, uploadVideo, + runServer, ServerInfo, setAccessTokensToServers, userLogin, updateCustomSubConfig } from '../../../../shared/utils' import { checkBadCountPagination, diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 42927605d..b9f05e952 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -62,6 +62,7 @@ function checkInitialConfig (data: CustomConfig) { expect(data.import.videos.http.enabled).to.be.true expect(data.import.videos.torrent.enabled).to.be.true + expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.false } function checkUpdatedConfig (data: CustomConfig) { @@ -103,6 +104,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.import.videos.http.enabled).to.be.false expect(data.import.videos.torrent.enabled).to.be.false + expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.true } describe('Test config', function () { @@ -225,6 +227,13 @@ describe('Test config', function () { enabled: false } } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: true + } + } } } await updateCustomConfig(server.url, server.accessToken, newCustomConfig) diff --git a/server/tests/api/users/user-notifications.ts b/server/tests/api/users/user-notifications.ts index d573bf024..1b66df79b 100644 --- a/server/tests/api/users/user-notifications.ts +++ b/server/tests/api/users/user-notifications.ts @@ -17,7 +17,9 @@ import { updateVideo, updateVideoChannel, userLogin, - wait + wait, + getCustomConfig, + updateCustomConfig } from '../../../../shared/utils' import { killallServers, ServerInfo, uploadVideo } from '../../../../shared/utils/index' import { setAccessTokensToServers } from '../../../../shared/utils/users/login' @@ -31,6 +33,7 @@ import { checkNewBlacklistOnMyVideo, checkNewCommentOnMyVideo, checkNewVideoAbuseForModerators, + checkVideoAutoBlacklistForModerators, checkNewVideoFromSubscription, checkUserRegistered, checkVideoIsPublished, @@ -54,6 +57,7 @@ import { getBadVideoUrl, getYoutubeVideoUrl, importVideo } from '../../../../sha import { addVideoCommentReply, addVideoCommentThread } from '../../../../shared/utils/videos/video-comments' import * as uuidv4 from 'uuid/v4' import { addAccountToAccountBlocklist, removeAccountFromAccountBlocklist } from '../../../../shared/utils/users/blocklist' +import { CustomConfig } from '../../../../shared/models/server' const expect = chai.expect @@ -92,6 +96,7 @@ describe('Test users notifications', function () { newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, newCommentOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, myVideoImportFinished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, myVideoPublished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, @@ -305,7 +310,7 @@ describe('Test users notifications', function () { }) it('Should send a new video notification after a video import', async function () { - this.timeout(30000) + this.timeout(100000) const name = 'video import ' + uuidv4() @@ -907,6 +912,180 @@ describe('Test users notifications', function () { }) }) + describe('Video-related notifications when video auto-blacklist is enabled', function () { + let userBaseParams: CheckerBaseParams + let adminBaseParamsServer1: CheckerBaseParams + let adminBaseParamsServer2: CheckerBaseParams + let videoUUID: string + let videoName: string + let currentCustomConfig: CustomConfig + + before(async () => { + + adminBaseParamsServer1 = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + + adminBaseParamsServer2 = { + server: servers[1], + emails, + socketNotifications: adminNotificationsServer2, + token: servers[1].accessToken + } + + userBaseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userAccessToken + } + + const resCustomConfig = await getCustomConfig(servers[0].url, servers[0].accessToken) + currentCustomConfig = resCustomConfig.body + const autoBlacklistTestsCustomConfig = immutableAssign(currentCustomConfig, { + autoBlacklist: { + videos: { + ofUsers: { + enabled: true + } + } + } + }) + // enable transcoding otherwise own publish notification after transcoding not expected + autoBlacklistTestsCustomConfig.transcoding.enabled = true + await updateCustomConfig(servers[0].url, servers[0].accessToken, autoBlacklistTestsCustomConfig) + + await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1_channel@localhost:9001') + await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1_channel@localhost:9001') + + }) + + it('Should send notification to moderators on new video with auto-blacklist', async function () { + this.timeout(20000) + + videoName = 'video with auto-blacklist ' + uuidv4() + const resVideo = await uploadVideo(servers[0].url, userAccessToken, { name: videoName }) + videoUUID = resVideo.body.video.uuid + + await waitJobs(servers) + await checkVideoAutoBlacklistForModerators(adminBaseParamsServer1, videoUUID, videoName, 'presence') + }) + + it('Should not send video publish notification if auto-blacklisted', async function () { + await checkVideoIsPublished(userBaseParams, videoName, videoUUID, 'absence') + }) + + it('Should not send a local user subscription notification if auto-blacklisted', async function () { + await checkNewVideoFromSubscription(adminBaseParamsServer1, videoName, videoUUID, 'absence') + }) + + it('Should not send a remote user subscription notification if auto-blacklisted', async function () { + await checkNewVideoFromSubscription(adminBaseParamsServer2, videoName, videoUUID, 'absence') + }) + + it('Should send video published and unblacklist after video unblacklisted', async function () { + this.timeout(20000) + + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, videoUUID) + + await waitJobs(servers) + + // FIXME: Can't test as two notifications sent to same user and util only checks last one + // One notification might be better anyways + // await checkNewBlacklistOnMyVideo(userBaseParams, videoUUID, videoName, 'unblacklist') + // await checkVideoIsPublished(userBaseParams, videoName, videoUUID, 'presence') + }) + + it('Should send a local user subscription notification after removed from blacklist', async function () { + await checkNewVideoFromSubscription(adminBaseParamsServer1, videoName, videoUUID, 'presence') + }) + + it('Should send a remote user subscription notification after removed from blacklist', async function () { + await checkNewVideoFromSubscription(adminBaseParamsServer2, videoName, videoUUID, 'presence') + }) + + it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { + this.timeout(20000) + + let updateAt = new Date(new Date().getTime() + 100000) + + const name = 'video with auto-blacklist and future schedule ' + uuidv4() + + const data = { + name, + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + const resVideo = await uploadVideo(servers[0].url, userAccessToken, data) + const uuid = resVideo.body.video.uuid + + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, uuid) + + await waitJobs(servers) + await checkNewBlacklistOnMyVideo(userBaseParams, uuid, name, 'unblacklist') + + // FIXME: Can't test absence as two notifications sent to same user and util only checks last one + // One notification might be better anyways + // await checkVideoIsPublished(userBaseParams, name, uuid, 'absence') + + await checkNewVideoFromSubscription(adminBaseParamsServer1, name, uuid, 'absence') + await checkNewVideoFromSubscription(adminBaseParamsServer2, name, uuid, 'absence') + }) + + it('Should not send publish/subscription notifications after scheduled update if video still auto-blacklisted', async function () { + this.timeout(20000) + + // In 2 seconds + let updateAt = new Date(new Date().getTime() + 2000) + + const name = 'video with schedule done and still auto-blacklisted ' + uuidv4() + + const data = { + name, + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + const resVideo = await uploadVideo(servers[0].url, userAccessToken, data) + const uuid = resVideo.body.video.uuid + + await wait(6000) + await checkVideoIsPublished(userBaseParams, name, uuid, 'absence') + await checkNewVideoFromSubscription(adminBaseParamsServer1, name, uuid, 'absence') + await checkNewVideoFromSubscription(adminBaseParamsServer2, name, uuid, 'absence') + }) + + it('Should not send a notification to moderators on new video without auto-blacklist', async function () { + this.timeout(20000) + + const name = 'video without auto-blacklist ' + uuidv4() + + // admin with blacklist right will not be auto-blacklisted + const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, { name }) + const uuid = resVideo.body.video.uuid + + await waitJobs(servers) + await checkVideoAutoBlacklistForModerators(adminBaseParamsServer1, uuid, name, 'absence') + }) + + after(async () => { + await updateCustomConfig(servers[0].url, servers[0].accessToken, currentCustomConfig) + + await removeUserSubscription(servers[0].url, servers[0].accessToken, 'user_1_channel@localhost:9001') + await removeUserSubscription(servers[1].url, servers[1].accessToken, 'user_1_channel@localhost:9001') + }) + }) + describe('Mark as read', function () { it('Should mark as read some notifications', async function () { const res = await getUserNotifications(servers[ 0 ].url, userAccessToken, 2, 3) @@ -968,7 +1147,7 @@ describe('Test users notifications', function () { }) it('Should not have notifications', async function () { - this.timeout(10000) + this.timeout(20000) await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.NONE @@ -987,7 +1166,7 @@ describe('Test users notifications', function () { }) it('Should only have web notifications', async function () { - this.timeout(10000) + this.timeout(20000) await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.WEB @@ -1013,7 +1192,7 @@ describe('Test users notifications', function () { }) it('Should only have mail notifications', async function () { - this.timeout(10000) + this.timeout(20000) await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.EMAIL @@ -1039,7 +1218,7 @@ describe('Test users notifications', function () { }) it('Should have email and web notifications', async function () { - this.timeout(10000) + this.timeout(20000) await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL diff --git a/server/tests/api/videos/video-blacklist.ts b/server/tests/api/videos/video-blacklist.ts index d39ad63b4..10b412a80 100644 --- a/server/tests/api/videos/video-blacklist.ts +++ b/server/tests/api/videos/video-blacklist.ts @@ -7,6 +7,7 @@ import { addVideoToBlacklist, flushAndRunMultipleServers, getBlacklistedVideosList, + getBlacklistedVideosListWithTypeFilter, getMyVideos, getSortedBlacklistedVideosList, getVideosList, @@ -22,7 +23,7 @@ import { } from '../../../../shared/utils/index' import { doubleFollow } from '../../../../shared/utils/server/follows' import { waitJobs } from '../../../../shared/utils/server/jobs' -import { VideoBlacklist } from '../../../../shared/models/videos' +import { VideoBlacklist, VideoBlacklistType } from '../../../../shared/models/videos' const expect = chai.expect @@ -101,7 +102,7 @@ describe('Test video blacklist management', function () { }) }) - describe('When listing blacklisted videos', function () { + describe('When listing manually blacklisted videos', function () { it('Should display all the blacklisted videos', async function () { const res = await getBlacklistedVideosList(servers[0].url, servers[0].accessToken) @@ -117,6 +118,26 @@ describe('Test video blacklist management', function () { } }) + it('Should display all the blacklisted videos when applying manual type filter', async function () { + const res = await getBlacklistedVideosListWithTypeFilter(servers[0].url, servers[0].accessToken, VideoBlacklistType.MANUAL) + + expect(res.body.total).to.equal(2) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + }) + + it('Should display nothing when applying automatic type filter', async function () { + const res = await getBlacklistedVideosListWithTypeFilter(servers[0].url, servers[0].accessToken, VideoBlacklistType.AUTO_BEFORE_PUBLISHED) // tslint:disable:max-line-length + + expect(res.body.total).to.equal(0) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(0) + }) + it('Should get the correct sort when sorting by descending id', async function () { const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-id') expect(res.body.total).to.equal(2) diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 20b261426..1607b40a8 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -77,4 +77,13 @@ export interface CustomConfig { } } } + + autoBlacklist: { + videos: { + ofUsers: { + enabled: boolean + } + } + } + } diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 0200d88ca..dcc45be8a 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -49,6 +49,14 @@ export interface ServerConfig { } } + autoBlacklist: { + videos: { + ofUsers: { + enabled: boolean + } + } + } + avatar: { file: { size: { diff --git a/shared/models/users/user-notification-setting.model.ts b/shared/models/users/user-notification-setting.model.ts index 531e12bba..57b33e4b8 100644 --- a/shared/models/users/user-notification-setting.model.ts +++ b/shared/models/users/user-notification-setting.model.ts @@ -8,6 +8,7 @@ export interface UserNotificationSetting { newVideoFromSubscription: UserNotificationSettingValue newCommentOnMyVideo: UserNotificationSettingValue videoAbuseAsModerator: UserNotificationSettingValue + videoAutoBlacklistAsModerator: UserNotificationSettingValue blacklistOnMyVideo: UserNotificationSettingValue myVideoPublished: UserNotificationSettingValue myVideoImportFinished: UserNotificationSettingValue diff --git a/shared/models/users/user-notification.model.ts b/shared/models/users/user-notification.model.ts index 186b62612..19892b61a 100644 --- a/shared/models/users/user-notification.model.ts +++ b/shared/models/users/user-notification.model.ts @@ -13,7 +13,9 @@ export enum UserNotificationType { NEW_USER_REGISTRATION = 9, NEW_FOLLOW = 10, - COMMENT_MENTION = 11 + COMMENT_MENTION = 11, + + VIDEO_AUTO_BLACKLIST_FOR_MODERATORS = 12 } export interface VideoInfo { diff --git a/shared/models/videos/blacklist/video-blacklist.model.ts b/shared/models/videos/blacklist/video-blacklist.model.ts index 4bd976190..68d59e489 100644 --- a/shared/models/videos/blacklist/video-blacklist.model.ts +++ b/shared/models/videos/blacklist/video-blacklist.model.ts @@ -1,19 +1,17 @@ +import { Video } from '../video.model' + +export enum VideoBlacklistType { + MANUAL = 1, + AUTO_BEFORE_PUBLISHED = 2 +} + export interface VideoBlacklist { id: number createdAt: Date updatedAt: Date unfederated: boolean reason?: string + type: VideoBlacklistType - video: { - id: number - name: string - uuid: string - description: string - duration: number - views: number - likes: number - dislikes: number - nsfw: boolean - } + video: Video } diff --git a/shared/utils/server/config.ts b/shared/utils/server/config.ts index 0e16af0f2..eaa493a93 100644 --- a/shared/utils/server/config.ts +++ b/shared/utils/server/config.ts @@ -112,6 +112,13 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) { enabled: false } } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } } } diff --git a/shared/utils/users/user-notifications.ts b/shared/utils/users/user-notifications.ts index c8ed7df30..e3a79f523 100644 --- a/shared/utils/users/user-notifications.ts +++ b/shared/utils/users/user-notifications.ts @@ -18,7 +18,7 @@ function updateMyNotificationSettings (url: string, token: string, settings: Use }) } -function getUserNotifications ( +async function getUserNotifications ( url: string, token: string, start: number, @@ -165,12 +165,15 @@ async function checkNewVideoFromSubscription (base: CheckerBaseParams, videoName checkVideo(notification.video, videoName, videoUUID) checkActor(notification.video.channel) } else { - expect(notification.video).to.satisfy(v => v === undefined || v.name !== videoName) + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.type !== UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION || n.video.name !== videoName + }) } } function emailFinder (email: object) { - return email[ 'text' ].indexOf(videoUUID) !== -1 + const text = email[ 'text' ] + return text.indexOf(videoUUID) !== -1 && text.indexOf('Your subscription') !== -1 } await checkNotification(base, notificationChecker, emailFinder, type) @@ -387,6 +390,31 @@ async function checkNewVideoAbuseForModerators (base: CheckerBaseParams, videoUU await checkNotification(base, notificationChecker, emailFinder, type) } +async function checkVideoAutoBlacklistForModerators (base: CheckerBaseParams, videoUUID: string, videoName: string, type: CheckerType) { + const notificationType = UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS + + function notificationChecker (notification: UserNotification, type: CheckerType) { + if (type === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.video.id).to.be.a('number') + checkVideo(notification.video, videoName, videoUUID) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.video === undefined || n.video.uuid !== videoUUID + }) + } + } + + function emailFinder (email: object) { + const text = email[ 'text' ] + return text.indexOf(videoUUID) !== -1 && email[ 'text' ].indexOf('video-auto-blacklist/list') !== -1 + } + + await checkNotification(base, notificationChecker, emailFinder, type) +} + async function checkNewBlacklistOnMyVideo ( base: CheckerBaseParams, videoUUID: string, @@ -431,6 +459,7 @@ export { checkCommentMention, updateMyNotificationSettings, checkNewVideoAbuseForModerators, + checkVideoAutoBlacklistForModerators, getUserNotifications, markAsReadNotifications, getLastNotification diff --git a/shared/utils/videos/video-blacklist.ts b/shared/utils/videos/video-blacklist.ts index f2ae0ed26..82d5b7e31 100644 --- a/shared/utils/videos/video-blacklist.ts +++ b/shared/utils/videos/video-blacklist.ts @@ -51,6 +51,18 @@ function getBlacklistedVideosList (url: string, token: string, specialStatus = 2 .expect('Content-Type', /json/) } +function getBlacklistedVideosListWithTypeFilter (url: string, token: string, type: number, specialStatus = 200) { + const path = '/api/v1/videos/blacklist/' + + return request(url) + .get(path) + .query({ sort: 'createdAt', type }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(specialStatus) + .expect('Content-Type', /json/) +} + function getSortedBlacklistedVideosList (url: string, token: string, sort: string, specialStatus = 200) { const path = '/api/v1/videos/blacklist/' @@ -69,6 +81,7 @@ export { addVideoToBlacklist, removeVideoFromBlacklist, getBlacklistedVideosList, + getBlacklistedVideosListWithTypeFilter, getSortedBlacklistedVideosList, updateVideoBlacklist } diff --git a/shared/utils/videos/video-change-ownership.ts b/shared/utils/videos/video-change-ownership.ts index f288692ea..371d02000 100644 --- a/shared/utils/videos/video-change-ownership.ts +++ b/shared/utils/videos/video-change-ownership.ts @@ -1,6 +1,6 @@ import * as request from 'supertest' -function changeVideoOwnership (url: string, token: string, videoId: number | string, username) { +function changeVideoOwnership (url: string, token: string, videoId: number | string, username, expectedStatus = 204) { const path = '/api/v1/videos/' + videoId + '/give-ownership' return request(url) @@ -8,7 +8,7 @@ function changeVideoOwnership (url: string, token: string, videoId: number | str .set('Accept', 'application/json') .set('Authorization', 'Bearer ' + token) .send({ username }) - .expect(204) + .expect(expectedStatus) } function getVideoChangeOwnershipList (url: string, token: string) {