interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add subdivision to viewer stats

This commit is contained in:
Chocobozzz 2023-12-28 09:12:20 +01:00
parent 4437ae0fd3
commit 4cbea51255
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
18 changed files with 243 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
client/src/app/+stats/video/video-stats.component.ts
View File

@ -18,11 +18,11 @@ import {
} from '@peertube/peertube-models'
import { VideoStatsService } from './video-stats.service'
type ActiveGraphId = VideoStatsTimeserieMetric | 'retention' | 'countries'
type ActiveGraphId = VideoStatsTimeserieMetric | 'retention' | 'countries' | 'regions'
type CountryData = { name: string, viewers: number }[]
type GeoData = { name: string, viewers: number }[]
type ChartIngestData = VideoStatsTimeserie | VideoStatsRetention | CountryData
type ChartIngestData = VideoStatsTimeserie | VideoStatsRetention | GeoData
type ChartBuilderResult = {
type: 'line' | 'bar'
@ -59,7 +59,8 @@ export class VideoStatsComponent implements OnInit {
video: VideoDetails
countries: CountryData = []
countries: GeoData = []
regions: GeoData = []
chartPlugins = [ zoomPlugin ]
@ -104,6 +105,11 @@ export class VideoStatsComponent implements OnInit {
id: 'countries',
label: $localize`Countries`,
zoomEnabled: false
},
{
id: 'regions',
label: $localize`Regions`,
zoomEnabled: false
}
]
@ -140,11 +146,17 @@ export class VideoStatsComponent implements OnInit {
return this.countries.length !== 0
}
hasRegions () {
return this.regions.length !== 0
}
onChartChange (newActive: ActiveGraphId) {
this.activeGraphId = newActive
if (newActive === 'countries') {
this.chartHeight = `${Math.max(this.countries.length * 20, 300)}px`
} else if (newActive === 'regions') {
this.chartHeight = `${Math.max(this.regions.length * 20, 300)}px`
} else {
this.chartHeight = '300px'
}
@ -193,6 +205,8 @@ export class VideoStatsComponent implements OnInit {
viewers: c.viewers
}))
this.regions = res.subdivisions
this.buildOverallStatCard(res)
},
@ -303,6 +317,13 @@ export class VideoStatsComponent implements OnInit {
value: this.numberFormatter.transform(overallStats.countries.length)
})
}
if (overallStats.subdivisions.length !== 0) {
this.overallStatCards.push({
label: $localize`Regions`,
value: this.numberFormatter.transform(overallStats.subdivisions.length)
})
}
}
private loadChart () {
@ -322,7 +343,9 @@ export class VideoStatsComponent implements OnInit {
metric: 'viewers'
}),
countries: of(this.countries)
countries: of(this.countries),
regions: of(this.regions)
}
obsBuilders[this.activeGraphId].subscribe({
@ -343,7 +366,8 @@ export class VideoStatsComponent implements OnInit {
retention: (rawData: VideoStatsRetention) => this.buildRetentionChartOptions(rawData),
aggregateWatchTime: (rawData: VideoStatsTimeserie) => this.buildTimeserieChartOptions(rawData),
viewers: (rawData: VideoStatsTimeserie) => this.buildTimeserieChartOptions(rawData),
countries: (rawData: CountryData) => this.buildCountryChartOptions(rawData)
countries: (rawData: GeoData) => this.buildGeoChartOptions(rawData),
regions: (rawData: GeoData) => this.buildGeoChartOptions(rawData)
}
const { type, data, displayLegend, plugins, options } = dataBuilders[graphId](this.chartIngestData[graphId])
@ -494,7 +518,7 @@ export class VideoStatsComponent implements OnInit {
}
}
private buildCountryChartOptions (rawData: CountryData): ChartBuilderResult {
private buildGeoChartOptions (rawData: GeoData): ChartBuilderResult {
const labels: string[] = []
const data: number[] = []
@ -574,7 +598,7 @@ export class VideoStatsComponent implements OnInit {
if (graphId === 'retention') return value + ' %'
if (graphId === 'aggregateWatchTime') return secondsToTime(+value)
if (graphId === 'countries' && scale) return scale.getLabelForValue(value as number)
if ((graphId === 'countries' || graphId === 'regions') && scale) return scale.getLabelForValue(value as number)
return value.toLocaleString(this.localeId)
}

2
client/src/app/core/rest/rest-table.ts
View File

@ -52,8 +52,6 @@ export abstract class RestTable <T = unknown> {
loadLazy (event: TableLazyLoadEvent) {
debugLogger('Load lazy %o.', event)
this.router.navigate([ '.' ], { relativeTo: this.route, queryParams: { start: event.first } })
this.sort = {
order: event.sortOrder,
field: event.sortField as string

3
config/default.yaml
View File

@ -383,6 +383,9 @@ geo_ip:
country:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
city:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb'
plugins:
# The website PeerTube will ask for available PeerTube plugins and themes
# This is an unmoderated plugin index, so only install plugins/themes you trust

3
config/production.yaml.example
View File

@ -381,6 +381,9 @@ geo_ip:
country:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
city:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb'
plugins:
# The website PeerTube will ask for available PeerTube plugins and themes
# This is an unmoderated plugin index, so only install plugins/themes you trust

1
packages/models/src/activitypub/objects/watch-action-object.ts
View File

@ -7,6 +7,7 @@ export interface WatchActionObject {
location?: {
addressCountry: string
addressRegion: string
}
uuid: string

5
packages/models/src/videos/stats/video-stats-overall.model.ts
View File

@ -11,4 +11,9 @@ export interface VideoStatsOverall {
isoCode: string
viewers: number
}[]
subdivisions: {
name: string
viewers: number
}[]
}

40
packages/tests/src/api/views/video-views-overall-stats.ts
View File

@ -305,10 +305,10 @@ describe('Test views overall stats', function () {
})
})
describe('Test countries', function () {
describe('Test countries/subdivisions', function () {
let videoUUID: string
it('Should not report countries if geoip is disabled', async function () {
it('Should not report countries/subdivisions if geoip is disabled', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
@ -320,9 +320,33 @@ describe('Test views overall stats', function () {
const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
expect(stats.countries).to.have.lengthOf(0)
expect(stats.subdivisions).to.have.lengthOf(0)
})
it('Should report countries if geoip is enabled', async function () {
it('Should not report subdivisions if database URL is not provided in the configuration', async function () {
this.timeout(240000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video without subdivisions' })
await waitJobs(servers)
await Promise.all([ servers[0].kill(), servers[1].kill() ])
const config = { geo_ip: { enabled: true, city: { database_url: '' } } }
await Promise.all([ servers[0].run(config), servers[1].run(config) ])
await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 })
await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 })
await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 })
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
expect(stats.countries).to.have.lengthOf(2)
expect(stats.subdivisions).to.have.lengthOf(0)
})
it('Should report countries/subdivisions if geoip is enabled', async function () {
this.timeout(240000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
@ -347,6 +371,7 @@ describe('Test views overall stats', function () {
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
expect(stats.countries).to.have.lengthOf(2)
expect(stats.countries[0].isoCode).to.equal('US')
@ -354,11 +379,18 @@ describe('Test views overall stats', function () {
expect(stats.countries[1].isoCode).to.equal('FR')
expect(stats.countries[1].viewers).to.equal(1)
expect(stats.subdivisions[0].name).to.equal('California')
expect(stats.subdivisions[0].viewers).to.equal(2)
expect(stats.subdivisions[1].name).to.equal('Brittany')
expect(stats.subdivisions[1].viewers).to.equal(1)
})
it('Should filter countries stats by date', async function () {
it('Should filter countries/subdivisions stats by date', async function () {
const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() })
expect(stats.countries).to.have.lengthOf(0)
expect(stats.subdivisions).to.have.lengthOf(0)
})
})

4
server/core/controllers/api/server/debug.ts
View File

@ -48,6 +48,10 @@ async function runCommand (req: express.Request, res: express.Response) {
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
}
if (!processors[body.command]) {
return res.fail({ message: 'Invalid command' })
}
await processors[body.command]()
return res.status(HttpStatusCode.NO_CONTENT_204).end()

6
server/core/helpers/custom-validators/activitypub/watch-action.ts
View File

@ -26,8 +26,12 @@ export {
function isLocationValid (location: any) {
if (!location) return true
if (typeof location !== 'object') return false
return typeof location === 'object' && typeof location.addressCountry === 'string'
if (location.addressCountry && typeof location.addressCountry !== 'string') return false
if (location.addressRegion && typeof location.addressRegion !== 'string') return false
return true
}
function isWatchSectionsValid (sections: WatchActionObject['watchSections']) {

118
server/core/helpers/geo-ip.ts
View File

@ -1,49 +1,96 @@
import { pathExists } from 'fs-extra/esm'
import { writeFile } from 'fs/promises'
import maxmind, { CountryResponse, Reader } from 'maxmind'
import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'
import { join } from 'path'
import { CONFIG } from '@server/initializers/config.js'
import { logger, loggerTagsFactory } from './logger.js'
import { isBinaryResponse, peertubeGot } from './requests.js'
import { isArray } from './custom-validators/misc.js'
const lTags = loggerTagsFactory('geo-ip')
const mmbdFilename = 'dbip-country-lite-latest.mmdb'
const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename)
export class GeoIP {
private static instance: GeoIP
private reader: Reader<CountryResponse>
private countryReader: Reader<CountryResponse>
private cityReader: Reader<CityResponse>
private readonly countryDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-country-lite-latest.mmdb')
private readonly cityDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-city-lite-latest.mmdb')
private constructor () {
}
async safeCountryISOLookup (ip: string): Promise<string> {
if (CONFIG.GEO_IP.ENABLED === false) return null
async safeIPISOLookup (ip: string): Promise<{ country: string, subdivisionName: string }> {
const emptyResult = { country: null, subdivisionName: null }
if (CONFIG.GEO_IP.ENABLED === false) return emptyResult
await this.initReaderIfNeeded()
await this.initReadersIfNeeded()
try {
const result = this.reader.get(ip)
if (!result) return null
const countryResult = this.countryReader?.get(ip)
const cityResult = this.cityReader?.get(ip)
return result.country.iso_code
return {
country: this.getISOCountry(countryResult),
subdivisionName: this.getISOSubdivision(cityResult)
}
} catch (err) {
logger.error('Cannot get country from IP.', { err })
logger.error('Cannot get country/city information from IP.', { err })
return null
return emptyResult
}
}
async updateDatabase () {
// ---------------------------------------------------------------------------
private getISOCountry (countryResult: CountryResponse) {
return countryResult?.country?.iso_code || null
}
private getISOSubdivision (subdivisionResult: CityResponse) {
const subdivisions = subdivisionResult?.subdivisions
if (!isArray(subdivisions) || subdivisions.length === 0) return null
// The last subdivision is the more precise one
const subdivision = subdivisions[subdivisions.length - 1]
return subdivision.names?.en || null
}
// ---------------------------------------------------------------------------
async updateDatabases () {
if (CONFIG.GEO_IP.ENABLED === false) return
const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL
await this.updateCountryDatabase()
await this.updateCityDatabase()
}
logger.info('Updating GeoIP database from %s.', url, lTags())
private async updateCountryDatabase () {
if (!CONFIG.GEO_IP.COUNTRY.DATABASE_URL) return false
const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' }
await this.updateDatabaseFile(CONFIG.GEO_IP.COUNTRY.DATABASE_URL, this.countryDBPath)
this.countryReader = undefined
return true
}
private async updateCityDatabase () {
if (!CONFIG.GEO_IP.CITY.DATABASE_URL) return false
await this.updateDatabaseFile(CONFIG.GEO_IP.CITY.DATABASE_URL, this.cityDBPath)
this.cityReader = undefined
return true
}
private async updateDatabaseFile (url: string, destination: string) {
logger.info('Updating GeoIP databases from %s.', url, lTags())
const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }
try {
const gotResult = await peertubeGot(url, gotOptions)
@ -52,27 +99,44 @@ export class GeoIP {
throw new Error('Not a binary response')
}
await writeFile(mmdbPath, gotResult.body)
await writeFile(destination, gotResult.body)
// Reinit reader
this.reader = undefined
logger.info('GeoIP database updated %s.', mmdbPath, lTags())
logger.info('GeoIP database updated %s.', destination, lTags())
} catch (err) {
logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
}
}
private async initReaderIfNeeded () {
if (!this.reader) {
if (!await pathExists(mmdbPath)) {
await this.updateDatabase()
// ---------------------------------------------------------------------------
private async initReadersIfNeeded () {
if (!this.countryReader) {
let open = true
if (!await pathExists(this.countryDBPath)) {
open = await this.updateCountryDatabase()
}
this.reader = await maxmind.open(mmdbPath)
if (open) {
this.countryReader = await maxmind.open(this.countryDBPath)
}
}
if (!this.cityReader) {
let open = true
if (!await pathExists(this.cityDBPath)) {
open = await this.updateCityDatabase()
}
if (open) {
this.cityReader = await maxmind.open(this.cityDBPath)
}
}
}
// ---------------------------------------------------------------------------
static get Instance () {
return this.instance || (this.instance = new this())
}

2
server/core/initializers/checker-before-init.ts
View File

@ -69,7 +69,7 @@ function checkMissedConfig () {
'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url',
'theme.default',
'feeds.videos.count', 'feeds.comments.count',
'geo_ip.enabled', 'geo_ip.country.database_url',
'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url',
'remote_redundancy.videos.accept_from',
'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions',
'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url',

3
server/core/initializers/config.ts
View File

@ -307,6 +307,9 @@ const CONFIG = {
ENABLED: config.get<boolean>('geo_ip.enabled'),
COUNTRY: {
DATABASE_URL: config.get<string>('geo_ip.country.database_url')
},
CITY: {
DATABASE_URL: config.get<string>('geo_ip.city.database_url')
}
},
PLUGINS: {

2
server/core/initializers/constants.ts
View File

@ -41,7 +41,7 @@ import { cpus } from 'os'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 800
const LAST_MIGRATION_VERSION = 805
// ---------------------------------------------------------------------------

27
server/core/initializers/migrations/0805-viewer-subdivision.ts Normal file
View File

@ -0,0 +1,27 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const { transaction } = utils
{
const data = {
type: Sequelize.STRING,
allowNull: true
}
await utils.queryInterface.addColumn('localVideoViewer', 'subdivisionName', data, { transaction })
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

5
server/core/lib/activitypub/local-video-viewer.ts
View File

@ -18,9 +18,8 @@ async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, v
startDate: new Date(watchAction.startTime),
endDate: new Date(watchAction.endTime),
country: watchAction.location
? watchAction.location.addressCountry
: null,
country: watchAction.location?.addressCountry || null,
subdivisionName: watchAction.location?.addressRegion || null,
videoId: video.id
}, { transaction: t })

2
server/core/lib/schedulers/geo-ip-update-scheduler.ts
View File

@ -13,7 +13,7 @@ export class GeoIPUpdateScheduler extends AbstractScheduler {
}
protected internalExecute () {
return GeoIP.Instance.updateDatabase()
return GeoIP.Instance.updateDatabases()
}
static get Instance () {

6
server/core/lib/views/shared/video-viewer-stats.ts
View File

@ -27,6 +27,7 @@ type LocalViewerStats = {
watchTime: number
country: string
subdivisionName: string
videoId: number
}
@ -85,7 +86,7 @@ export class VideoViewerStats {
}
if (!stats) {
const country = await GeoIP.Instance.safeCountryISOLookup(ip)
const { country, subdivisionName } = await GeoIP.Instance.safeIPISOLookup(ip)
stats = {
firstUpdated: nowMs,
@ -96,6 +97,8 @@ export class VideoViewerStats {
watchTime: 0,
country,
subdivisionName,
videoId: video.id
}
}
@ -180,6 +183,7 @@ export class VideoViewerStats {
endDate: new Date(stats.lastUpdated),
watchTime: stats.watchTime,
country: stats.country,
subdivisionName: stats.subdivisionName,
videoId: video.id
})

35
server/core/models/view/local-video-viewer.ts
View File

@ -54,6 +54,10 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
@Column
country: string
@AllowNull(true)
@Column
subdivisionName: string
@AllowNull(false)
@Default(DataType.UUIDV4)
@IsUUID(4)
@ -199,26 +203,27 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
return LocalVideoViewerModel.sequelize.query<any>(watchPeakQuery, queryOptions)
}
const buildCountriesPromise = () => {
let countryDateWhere = ''
const buildGeoPromise = (type: 'country' | 'subdivisionName') => {
let dateWhere = ''
if (startDate) countryDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate'
if (endDate) countryDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate'
if (startDate) dateWhere += ' AND "localVideoViewer"."endDate" >= :startDate'
if (endDate) dateWhere += ' AND "localVideoViewer"."startDate" <= :endDate'
const countriesQuery = `SELECT country, COUNT(country) as viewers ` +
const query = `SELECT "${type}", COUNT("${type}") as viewers ` +
`FROM "localVideoViewer" ` +
`WHERE "videoId" = :videoId AND country IS NOT NULL ${countryDateWhere} ` +
`GROUP BY country ` +
`ORDER BY viewers DESC`
`WHERE "videoId" = :videoId AND "${type}" IS NOT NULL ${dateWhere} ` +
`GROUP BY "${type}" ` +
`ORDER BY "viewers" DESC`
return LocalVideoViewerModel.sequelize.query<any>(countriesQuery, queryOptions)
return LocalVideoViewerModel.sequelize.query<any>(query, queryOptions)
}
const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries ] = await Promise.all([
const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries, rowsSubdivisions ] = await Promise.all([
buildTotalViewersPromise(),
buildWatchTimePromise(),
buildWatchPeakPromise(),
buildCountriesPromise()
buildGeoPromise('country'),
buildGeoPromise('subdivisionName')
])
const viewersPeak = rowsWatchPeak.length !== 0
@ -245,6 +250,11 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
countries: rowsCountries.map(r => ({
isoCode: r.country,
viewers: r.viewers
})),
subdivisions: rowsSubdivisions.map(r => ({
name: r.subdivisionName,
viewers: r.viewers
}))
}
}
@ -347,7 +357,8 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
const location = this.country
? {
location: {
addressCountry: this.country
addressCountry: this.country,
addressRegion: this.subdivisionName
}
}
: {}