From fe19f600dab0f6b00a7aa146ba4bd4bb96536155 Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Mon, 12 Apr 2021 11:19:07 +0200 Subject: [PATCH] add channel and playlist stats to server stats endpoint (#3747) * add channel and playlist stats to nodeinfo * add tests for active video channels stats * fix tests for active channel stats --- server/lib/stat-manager.ts | 31 +++++++-- server/models/video/video-channel.ts | 43 +++++++++++- server/models/video/video-playlist.ts | 60 +++++++++++++---- server/tests/api/server/stats.ts | 76 +++++++++++++++++++++- shared/models/server/server-stats.model.ts | 7 ++ 5 files changed, 193 insertions(+), 24 deletions(-) diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts index 547d7a56b..09ba208bd 100644 --- a/server/lib/stat-manager.ts +++ b/server/lib/stat-manager.ts @@ -3,8 +3,10 @@ import { UserModel } from '@server/models/account/user' import { ActorFollowModel } from '@server/models/activitypub/actor-follow' import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' import { VideoModel } from '@server/models/video/video' +import { VideoChannelModel } from '@server/models/video/video-channel' import { VideoCommentModel } from '@server/models/video/video-comment' import { VideoFileModel } from '@server/models/video/video-file' +import { VideoPlaylistModel } from '@server/models/video/video-playlist' import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models' class StatsManager { @@ -46,22 +48,37 @@ class StatsManager { const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() + const { + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels + } = await VideoChannelModel.getStats() + const { totalLocalPlaylists } = await VideoPlaylistModel.getStats() const videosRedundancyStats = await this.buildRedundancyStats() const data: ServerStats = { - totalLocalVideos, - totalLocalVideoViews, - totalLocalVideoFilesSize, - totalLocalVideoComments, - totalVideos, - totalVideoComments, - totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers, + totalLocalVideos, + totalLocalVideoViews, + totalLocalVideoComments, + totalLocalVideoFilesSize, + + totalVideos, + totalVideoComments, + + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels, + + totalLocalPlaylists, + totalInstanceFollowers, totalInstanceFollowing, diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index d2a055f5b..b7ffbd3b1 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -1,4 +1,4 @@ -import { FindOptions, Includeable, literal, Op, ScopeOptions } from 'sequelize' +import { FindOptions, Includeable, literal, Op, QueryTypes, ScopeOptions } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -338,6 +338,47 @@ export class VideoChannelModel extends Model { return VideoChannelModel.count(query) } + static async getStats () { + + function getActiveVideoChannels (days: number) { + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + raw: true + } + + const query = ` +SELECT COUNT(DISTINCT("VideoChannelModel"."id")) AS "count" +FROM "videoChannel" AS "VideoChannelModel" +INNER JOIN "video" AS "Videos" +ON "VideoChannelModel"."id" = "Videos"."channelId" +AND ("Videos"."publishedAt" > Now() - interval '${days}d') +INNER JOIN "account" AS "Account" +ON "VideoChannelModel"."accountId" = "Account"."id" +INNER JOIN "actor" AS "Account->Actor" +ON "Account"."actorId" = "Account->Actor"."id" +AND "Account->Actor"."serverId" IS NULL +LEFT OUTER JOIN "server" AS "Account->Actor->Server" +ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"` + + return VideoChannelModel.sequelize.query<{ count: string }>(query, options) + .then(r => parseInt(r[0].count, 10)) + } + + const totalLocalVideoChannels = await VideoChannelModel.count() + const totalLocalDailyActiveVideoChannels = await getActiveVideoChannels(1) + const totalLocalWeeklyActiveVideoChannels = await getActiveVideoChannels(7) + const totalLocalMonthlyActiveVideoChannels = await getActiveVideoChannels(30) + const totalHalfYearActiveVideoChannels = await getActiveVideoChannels(180) + + return { + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels, + totalHalfYearActiveVideoChannels + } + } + static listForApi (parameters: { actorId: number start: number diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 49a406608..efe5be36d 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -54,6 +54,7 @@ import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getPlaylistSort, isOutdat import { ThumbnailModel } from './thumbnail' import { ScopeNames as VideoChannelScopeNames, VideoChannelModel } from './video-channel' import { VideoPlaylistElementModel } from './video-playlist-element' +import { ActorModel } from '../activitypub/actor' enum ScopeNames { AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST', @@ -65,7 +66,7 @@ enum ScopeNames { } type AvailableForListOptions = { - followerActorId: number + followerActorId?: number type?: VideoPlaylistType accountId?: number videoChannelId?: number @@ -134,20 +135,26 @@ type AvailableForListOptions = { privacy: VideoPlaylistPrivacy.PUBLIC }) - // Only list local playlists OR playlists that are on an instance followed by actorId - const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId) + // Only list local playlists + const whereActorOr: WhereOptions[] = [ + { + serverId: null + } + ] + + // … OR playlists that are on an instance followed by actorId + if (options.followerActorId) { + const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId) + + whereActorOr.push({ + serverId: { + [Op.in]: literal(inQueryInstanceFollow) + } + }) + } whereActor = { - [Op.or]: [ - { - serverId: null - }, - { - serverId: { - [Op.in]: literal(inQueryInstanceFollow) - } - } - ] + [Op.or]: whereActorOr } } @@ -495,6 +502,33 @@ export class VideoPlaylistModel extends Model { return '/video-playlists/embed/' + this.uuid } + static async getStats () { + const totalLocalPlaylists = await VideoPlaylistModel.count({ + include: [ + { + model: AccountModel, + required: true, + include: [ + { + model: ActorModel, + required: true, + where: { + serverId: null + } + } + ] + } + ], + where: { + privacy: VideoPlaylistPrivacy.PUBLIC + } + }) + + return { + totalLocalPlaylists + } + } + setAsRefreshed () { this.changed('updatedAt', true) diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index eb474c1f5..304181a6d 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -3,8 +3,10 @@ import 'mocha' import * as chai from 'chai' import { + addVideoChannel, cleanupTests, createUser, + createVideoPlaylist, doubleFollow, flushAndRunMultipleServers, follow, @@ -21,12 +23,14 @@ import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import { getStats } from '../../../../shared/extra-utils/server/stats' import { addVideoCommentThread } from '../../../../shared/extra-utils/videos/video-comments' import { ServerStats } from '../../../../shared/models/server/server-stats.model' +import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' import { ActivityType } from '@shared/models' const expect = chai.expect describe('Test stats (excluding redundancy)', function () { let servers: ServerInfo[] = [] + let channelId const user = { username: 'user1', password: 'super_password' @@ -70,6 +74,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(2) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalLocalPlaylists).to.equal(0) }) it('Should have the correct stats on instance 2', async function () { @@ -85,6 +90,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalLocalPlaylists).to.equal(0) }) it('Should have the correct stats on instance 3', async function () { @@ -99,6 +105,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowers).to.equal(0) + expect(data.totalLocalPlaylists).to.equal(0) }) it('Should have the correct total videos stats after an unfollow', async function () { @@ -113,7 +120,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(0) }) - it('Should have the correct active users stats', async function () { + it('Should have the correct active user stats', async function () { const server = servers[0] { @@ -135,6 +142,69 @@ describe('Test stats (excluding redundancy)', function () { } }) + it('Should have the correct active channel stats', async function () { + const server = servers[0] + + { + const res = await getStats(server.url) + const data: ServerStats = res.body + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } + + { + const channelAttributes = { + name: 'stats_channel', + displayName: 'My stats channel' + } + const resChannel = await addVideoChannel(server.url, server.accessToken, channelAttributes) + channelId = resChannel.body.videoChannel.id + + const res = await getStats(server.url) + const data: ServerStats = res.body + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } + + { + await uploadVideo(server.url, server.accessToken, { fixture: 'video_short.webm', channelId }) + + const res = await getStats(server.url) + const data: ServerStats = res.body + expect(data.totalLocalDailyActiveVideoChannels).to.equal(2) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2) + } + }) + + it('Should have the correct playlist stats', async function () { + const server = servers[0] + + { + const resStats = await getStats(server.url) + const dataStats: ServerStats = resStats.body + expect(dataStats.totalLocalPlaylists).to.equal(0) + } + + { + await createVideoPlaylist({ + url: server.url, + token: server.accessToken, + playlistAttrs: { + displayName: 'playlist for count', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: channelId + } + }) + + const resStats = await getStats(server.url) + const dataStats: ServerStats = resStats.body + expect(dataStats.totalLocalPlaylists).to.equal(1) + } + }) + it('Should correctly count video file sizes if transcoding is enabled', async function () { this.timeout(60000) @@ -173,8 +243,8 @@ describe('Test stats (excluding redundancy)', function () { { const res = await getStats(servers[0].url) const data: ServerStats = res.body - expect(data.totalLocalVideoFilesSize).to.be.greaterThan(300000) - expect(data.totalLocalVideoFilesSize).to.be.lessThan(400000) + expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000) + expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000) } }) diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts index 0f8cfc6cf..b1dcf2065 100644 --- a/shared/models/server/server-stats.model.ts +++ b/shared/models/server/server-stats.model.ts @@ -13,6 +13,13 @@ export interface ServerStats { totalVideos: number totalVideoComments: number + totalLocalVideoChannels: number + totalLocalDailyActiveVideoChannels: number + totalLocalWeeklyActiveVideoChannels: number + totalLocalMonthlyActiveVideoChannels: number + + totalLocalPlaylists: number + totalInstanceFollowers: number totalInstanceFollowing: number