Add subdivision to viewer stats

client/src/app/+stats/video/video-stats.component.ts

@@ -18,11 +18,11 @@ import {
 } from '@peertube/peertube-models'
 import { VideoStatsService } from './video-stats.service'
 
-type ActiveGraphId = VideoStatsTimeserieMetric | 'retention' | 'countries'
+type ActiveGraphId = VideoStatsTimeserieMetric | 'retention' | 'countries' | 'regions'
 
-type CountryData = { name: string, viewers: number }[]
+type GeoData = { name: string, viewers: number }[]
 
-type ChartIngestData = VideoStatsTimeserie | VideoStatsRetention | CountryData
+type ChartIngestData = VideoStatsTimeserie | VideoStatsRetention | GeoData
 type ChartBuilderResult = {
   type: 'line' | 'bar'
 
@@ -59,7 +59,8 @@ export class VideoStatsComponent implements OnInit {
 
   video: VideoDetails
 
-  countries: CountryData = []
+  countries: GeoData = []
+  regions: GeoData = []
 
   chartPlugins = [ zoomPlugin ]
 
@@ -104,6 +105,11 @@ export class VideoStatsComponent implements OnInit {
         id: 'countries',
         label: $localize`Countries`,
         zoomEnabled: false
+      },
+      {
+        id: 'regions',
+        label: $localize`Regions`,
+        zoomEnabled: false
       }
     ]
 
@@ -140,11 +146,17 @@ export class VideoStatsComponent implements OnInit {
     return this.countries.length !== 0
   }
 
+  hasRegions () {
+    return this.regions.length !== 0
+  }
+
   onChartChange (newActive: ActiveGraphId) {
     this.activeGraphId = newActive
 
     if (newActive === 'countries') {
       this.chartHeight = `${Math.max(this.countries.length * 20, 300)}px`
+    } else if (newActive === 'regions') {
+      this.chartHeight = `${Math.max(this.regions.length * 20, 300)}px`
     } else {
       this.chartHeight = '300px'
     }
@@ -193,6 +205,8 @@ export class VideoStatsComponent implements OnInit {
             viewers: c.viewers
           }))
 
+          this.regions = res.subdivisions
+
           this.buildOverallStatCard(res)
         },
 
@@ -303,6 +317,13 @@ export class VideoStatsComponent implements OnInit {
         value: this.numberFormatter.transform(overallStats.countries.length)
       })
     }
+
+    if (overallStats.subdivisions.length !== 0) {
+      this.overallStatCards.push({
+        label: $localize`Regions`,
+        value: this.numberFormatter.transform(overallStats.subdivisions.length)
+      })
+    }
   }
 
   private loadChart () {
@@ -322,7 +343,9 @@ export class VideoStatsComponent implements OnInit {
         metric: 'viewers'
       }),
 
-      countries: of(this.countries)
+      countries: of(this.countries),
+
+      regions: of(this.regions)
     }
 
     obsBuilders[this.activeGraphId].subscribe({
@@ -343,7 +366,8 @@ export class VideoStatsComponent implements OnInit {
       retention: (rawData: VideoStatsRetention) => this.buildRetentionChartOptions(rawData),
       aggregateWatchTime: (rawData: VideoStatsTimeserie) => this.buildTimeserieChartOptions(rawData),
       viewers: (rawData: VideoStatsTimeserie) => this.buildTimeserieChartOptions(rawData),
-      countries: (rawData: CountryData) => this.buildCountryChartOptions(rawData)
+      countries: (rawData: GeoData) => this.buildGeoChartOptions(rawData),
+      regions: (rawData: GeoData) => this.buildGeoChartOptions(rawData)
     }
 
     const { type, data, displayLegend, plugins, options } = dataBuilders[graphId](this.chartIngestData[graphId])
@@ -494,7 +518,7 @@ export class VideoStatsComponent implements OnInit {
     }
   }
 
-  private buildCountryChartOptions (rawData: CountryData): ChartBuilderResult {
+  private buildGeoChartOptions (rawData: GeoData): ChartBuilderResult {
     const labels: string[] = []
     const data: number[] = []
 
@@ -574,7 +598,7 @@ export class VideoStatsComponent implements OnInit {
 
     if (graphId === 'retention') return value + ' %'
     if (graphId === 'aggregateWatchTime') return secondsToTime(+value)
-    if (graphId === 'countries' && scale) return scale.getLabelForValue(value as number)
+    if ((graphId === 'countries' || graphId === 'regions') && scale) return scale.getLabelForValue(value as number)
 
     return value.toLocaleString(this.localeId)
   }

client/src/app/core/rest/rest-table.ts

@@ -52,8 +52,6 @@ export abstract class RestTable <T = unknown> {
   loadLazy (event: TableLazyLoadEvent) {
     debugLogger('Load lazy %o.', event)
 
-    this.router.navigate([ '.' ], { relativeTo: this.route, queryParams: { start: event.first } })
-
     this.sort = {
       order: event.sortOrder,
       field: event.sortField as string

config/default.yaml

@@ -383,6 +383,9 @@ geo_ip:
   country:
     database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
 
+  city:
+    database_url: 'https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb'
+
 plugins:
   # The website PeerTube will ask for available PeerTube plugins and themes
   # This is an unmoderated plugin index, so only install plugins/themes you trust

config/production.yaml.example

@@ -381,6 +381,9 @@ geo_ip:
   country:
     database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
 
+  city:
+    database_url: 'https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb'
+
 plugins:
   # The website PeerTube will ask for available PeerTube plugins and themes
   # This is an unmoderated plugin index, so only install plugins/themes you trust

packages/models/src/activitypub/objects/watch-action-object.ts

@@ -7,6 +7,7 @@ export interface WatchActionObject {
 
   location?: {
     addressCountry: string
+    addressRegion: string
   }
 
   uuid: string

packages/models/src/videos/stats/video-stats-overall.model.ts

@@ -11,4 +11,9 @@ export interface VideoStatsOverall {
     isoCode: string
     viewers: number
   }[]
+
+  subdivisions: {
+    name: string
+    viewers: number
+  }[]
 }

packages/tests/src/api/views/video-views-overall-stats.ts

@@ -305,10 +305,10 @@ describe('Test views overall stats', function () {
     })
   })
 
-  describe('Test countries', function () {
+  describe('Test countries/subdivisions', function () {
     let videoUUID: string
 
-    it('Should not report countries if geoip is disabled', async function () {
+    it('Should not report countries/subdivisions if geoip is disabled', async function () {
       this.timeout(120000)
 
       const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
@@ -320,9 +320,33 @@ describe('Test views overall stats', function () {
 
       const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
       expect(stats.countries).to.have.lengthOf(0)
+      expect(stats.subdivisions).to.have.lengthOf(0)
     })
 
-    it('Should report countries if geoip is enabled', async function () {
+    it('Should not report subdivisions if database URL is not provided in the configuration', async function () {
+      this.timeout(240000)
+
+      const { uuid } = await servers[0].videos.quickUpload({ name: 'video without subdivisions' })
+      await waitJobs(servers)
+
+      await Promise.all([ servers[0].kill(), servers[1].kill() ])
+
+      const config = { geo_ip: { enabled: true, city: { database_url: '' } } }
+      await Promise.all([ servers[0].run(config), servers[1].run(config) ])
+
+      await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 })
+      await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 })
+      await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 })
+
+      await processViewersStats(servers)
+
+      const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
+
+      expect(stats.countries).to.have.lengthOf(2)
+      expect(stats.subdivisions).to.have.lengthOf(0)
+    })
+
+    it('Should report countries/subdivisions if geoip is enabled', async function () {
       this.timeout(240000)
 
       const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
@@ -347,6 +371,7 @@ describe('Test views overall stats', function () {
       await processViewersStats(servers)
 
       const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
+
       expect(stats.countries).to.have.lengthOf(2)
 
       expect(stats.countries[0].isoCode).to.equal('US')
@@ -354,11 +379,18 @@ describe('Test views overall stats', function () {
 
       expect(stats.countries[1].isoCode).to.equal('FR')
       expect(stats.countries[1].viewers).to.equal(1)
+
+      expect(stats.subdivisions[0].name).to.equal('California')
+      expect(stats.subdivisions[0].viewers).to.equal(2)
+
+      expect(stats.subdivisions[1].name).to.equal('Brittany')
+      expect(stats.subdivisions[1].viewers).to.equal(1)
     })
 
-    it('Should filter countries stats by date', async function () {
+    it('Should filter countries/subdivisions stats by date', async function () {
       const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() })
       expect(stats.countries).to.have.lengthOf(0)
+      expect(stats.subdivisions).to.have.lengthOf(0)
     })
   })
 

server/core/controllers/api/server/debug.ts

@@ -48,6 +48,10 @@ async function runCommand (req: express.Request, res: express.Response) {
     'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
   }
 
+  if (!processors[body.command]) {
+    return res.fail({ message: 'Invalid command' })
+  }
+
   await processors[body.command]()
 
   return res.status(HttpStatusCode.NO_CONTENT_204).end()

server/core/helpers/custom-validators/activitypub/watch-action.ts

@@ -26,8 +26,12 @@ export {
 
 function isLocationValid (location: any) {
   if (!location) return true
+  if (typeof location !== 'object') return false
 
-  return typeof location === 'object' && typeof location.addressCountry === 'string'
+  if (location.addressCountry && typeof location.addressCountry !== 'string') return false
+  if (location.addressRegion && typeof location.addressRegion !== 'string') return false
+
+  return true
 }
 
 function isWatchSectionsValid (sections: WatchActionObject['watchSections']) {

server/core/helpers/geo-ip.ts

@@ -1,49 +1,96 @@
 import { pathExists } from 'fs-extra/esm'
 import { writeFile } from 'fs/promises'
-import maxmind, { CountryResponse, Reader } from 'maxmind'
+import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'
 import { join } from 'path'
 import { CONFIG } from '@server/initializers/config.js'
 import { logger, loggerTagsFactory } from './logger.js'
 import { isBinaryResponse, peertubeGot } from './requests.js'
+import { isArray } from './custom-validators/misc.js'
 
 const lTags = loggerTagsFactory('geo-ip')
 
-const mmbdFilename = 'dbip-country-lite-latest.mmdb'
-const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename)
-
 export class GeoIP {
   private static instance: GeoIP
 
-  private reader: Reader<CountryResponse>
+  private countryReader: Reader<CountryResponse>
+  private cityReader: Reader<CityResponse>
+
+  private readonly countryDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-country-lite-latest.mmdb')
+  private readonly cityDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-city-lite-latest.mmdb')
 
   private constructor () {
   }
 
-  async safeCountryISOLookup (ip: string): Promise<string> {
+  async safeIPISOLookup (ip: string): Promise<{ country: string, subdivisionName: string }> {
-    if (CONFIG.GEO_IP.ENABLED === false) return null
+    const emptyResult = { country: null, subdivisionName: null }
+    if (CONFIG.GEO_IP.ENABLED === false) return emptyResult
 
-    await this.initReaderIfNeeded()
+    await this.initReadersIfNeeded()
 
     try {
-      const result = this.reader.get(ip)
+      const countryResult = this.countryReader?.get(ip)
-      if (!result) return null
+      const cityResult = this.cityReader?.get(ip)
 
-      return result.country.iso_code
+      return {
+        country: this.getISOCountry(countryResult),
+        subdivisionName: this.getISOSubdivision(cityResult)
+      }
     } catch (err) {
-      logger.error('Cannot get country from IP.', { err })
+      logger.error('Cannot get country/city information from IP.', { err })
 
-      return null
+      return emptyResult
     }
   }
 
-  async updateDatabase () {
+  // ---------------------------------------------------------------------------
+
+  private getISOCountry (countryResult: CountryResponse) {
+    return countryResult?.country?.iso_code || null
+  }
+
+  private getISOSubdivision (subdivisionResult: CityResponse) {
+    const subdivisions = subdivisionResult?.subdivisions
+    if (!isArray(subdivisions) || subdivisions.length === 0) return null
+
+    // The last subdivision is the more precise one
+    const subdivision = subdivisions[subdivisions.length - 1]
+
+    return subdivision.names?.en || null
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async updateDatabases () {
     if (CONFIG.GEO_IP.ENABLED === false) return
 
-    const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL
+    await this.updateCountryDatabase()
+    await this.updateCityDatabase()
+  }
 
-    logger.info('Updating GeoIP database from %s.', url, lTags())
+  private async updateCountryDatabase () {
+    if (!CONFIG.GEO_IP.COUNTRY.DATABASE_URL) return false
 
-    const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' }
+    await this.updateDatabaseFile(CONFIG.GEO_IP.COUNTRY.DATABASE_URL, this.countryDBPath)
+
+    this.countryReader = undefined
+
+    return true
+  }
+
+  private async updateCityDatabase () {
+    if (!CONFIG.GEO_IP.CITY.DATABASE_URL) return false
+
+    await this.updateDatabaseFile(CONFIG.GEO_IP.CITY.DATABASE_URL, this.cityDBPath)
+
+    this.cityReader = undefined
+
+    return true
+  }
+
+  private async updateDatabaseFile (url: string, destination: string) {
+    logger.info('Updating GeoIP databases from %s.', url, lTags())
+
+    const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }
 
     try {
       const gotResult = await peertubeGot(url, gotOptions)
@@ -52,27 +99,44 @@ export class GeoIP {
         throw new Error('Not a binary response')
       }
 
-      await writeFile(mmdbPath, gotResult.body)
+      await writeFile(destination, gotResult.body)
 
-      // Reinit reader
+      logger.info('GeoIP database updated %s.', destination, lTags())
-      this.reader = undefined
-
-      logger.info('GeoIP database updated %s.', mmdbPath, lTags())
     } catch (err) {
       logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
     }
   }
 
-  private async initReaderIfNeeded () {
+  // ---------------------------------------------------------------------------
-    if (!this.reader) {
+
-      if (!await pathExists(mmdbPath)) {
+  private async initReadersIfNeeded () {
-        await this.updateDatabase()
+    if (!this.countryReader) {
+      let open = true
+
+      if (!await pathExists(this.countryDBPath)) {
+        open = await this.updateCountryDatabase()
       }
 
-      this.reader = await maxmind.open(mmdbPath)
+      if (open) {
+        this.countryReader = await maxmind.open(this.countryDBPath)
+      }
+    }
+
+    if (!this.cityReader) {
+      let open = true
+
+      if (!await pathExists(this.cityDBPath)) {
+        open = await this.updateCityDatabase()
+      }
+
+      if (open) {
+        this.cityReader = await maxmind.open(this.cityDBPath)
+      }
     }
   }
 
+  // ---------------------------------------------------------------------------
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }

server/core/initializers/checker-before-init.ts

@@ -69,7 +69,7 @@ function checkMissedConfig () {
     'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url',
     'theme.default',
     'feeds.videos.count', 'feeds.comments.count',
-    'geo_ip.enabled', 'geo_ip.country.database_url',
+    'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url',
     'remote_redundancy.videos.accept_from',
     'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions',
     'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url',

server/core/initializers/config.ts

@@ -307,6 +307,9 @@ const CONFIG = {
     ENABLED: config.get<boolean>('geo_ip.enabled'),
     COUNTRY: {
       DATABASE_URL: config.get<string>('geo_ip.country.database_url')
+    },
+    CITY: {
+      DATABASE_URL: config.get<string>('geo_ip.city.database_url')
     }
   },
   PLUGINS: {

server/core/initializers/constants.ts

@@ -41,7 +41,7 @@ import { cpus } from 'os'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 800
+const LAST_MIGRATION_VERSION = 805
 
 // ---------------------------------------------------------------------------
 

server/core/initializers/migrations/0805-viewer-subdivision.ts

@@ -0,0 +1,27 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  const { transaction } = utils
+
+  {
+    const data = {
+      type: Sequelize.STRING,
+      allowNull: true
+    }
+
+    await utils.queryInterface.addColumn('localVideoViewer', 'subdivisionName', data, { transaction })
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/core/lib/activitypub/local-video-viewer.ts

@@ -18,9 +18,8 @@ async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, v
     startDate: new Date(watchAction.startTime),
     endDate: new Date(watchAction.endTime),
 
-    country: watchAction.location
+    country: watchAction.location?.addressCountry || null,
-      ? watchAction.location.addressCountry
+    subdivisionName: watchAction.location?.addressRegion || null,
-      : null,
 
     videoId: video.id
   }, { transaction: t })

server/core/lib/schedulers/geo-ip-update-scheduler.ts

@@ -13,7 +13,7 @@ export class GeoIPUpdateScheduler extends AbstractScheduler {
   }
 
   protected internalExecute () {
-    return GeoIP.Instance.updateDatabase()
+    return GeoIP.Instance.updateDatabases()
   }
 
   static get Instance () {

server/core/lib/views/shared/video-viewer-stats.ts

@@ -27,6 +27,7 @@ type LocalViewerStats = {
   watchTime: number
 
   country: string
+  subdivisionName: string
 
   videoId: number
 }
@@ -85,7 +86,7 @@ export class VideoViewerStats {
     }
 
     if (!stats) {
-      const country = await GeoIP.Instance.safeCountryISOLookup(ip)
+      const { country, subdivisionName } = await GeoIP.Instance.safeIPISOLookup(ip)
 
       stats = {
         firstUpdated: nowMs,
@@ -96,6 +97,8 @@ export class VideoViewerStats {
         watchTime: 0,
 
         country,
+        subdivisionName,
+
         videoId: video.id
       }
     }
@@ -180,6 +183,7 @@ export class VideoViewerStats {
       endDate: new Date(stats.lastUpdated),
       watchTime: stats.watchTime,
       country: stats.country,
+      subdivisionName: stats.subdivisionName,
       videoId: video.id
     })
 

server/core/models/view/local-video-viewer.ts

@@ -54,6 +54,10 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
   @Column
   country: string
 
+  @AllowNull(true)
+  @Column
+  subdivisionName: string
+
   @AllowNull(false)
   @Default(DataType.UUIDV4)
   @IsUUID(4)
@@ -199,26 +203,27 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
       return LocalVideoViewerModel.sequelize.query<any>(watchPeakQuery, queryOptions)
     }
 
-    const buildCountriesPromise = () => {
+    const buildGeoPromise = (type: 'country' | 'subdivisionName') => {
-      let countryDateWhere = ''
+      let dateWhere = ''
 
-      if (startDate) countryDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate'
+      if (startDate) dateWhere += ' AND "localVideoViewer"."endDate" >= :startDate'
-      if (endDate) countryDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate'
+      if (endDate) dateWhere += ' AND "localVideoViewer"."startDate" <= :endDate'
 
-      const countriesQuery = `SELECT country, COUNT(country) as viewers ` +
+      const query = `SELECT "${type}", COUNT("${type}") as viewers ` +
         `FROM "localVideoViewer" ` +
-        `WHERE "videoId" = :videoId AND country IS NOT NULL ${countryDateWhere} ` +
+        `WHERE "videoId" = :videoId AND "${type}" IS NOT NULL ${dateWhere} ` +
-        `GROUP BY country ` +
+        `GROUP BY "${type}" ` +
-        `ORDER BY viewers DESC`
+        `ORDER BY "viewers" DESC`
 
-      return LocalVideoViewerModel.sequelize.query<any>(countriesQuery, queryOptions)
+      return LocalVideoViewerModel.sequelize.query<any>(query, queryOptions)
     }
 
-    const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries ] = await Promise.all([
+    const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries, rowsSubdivisions ] = await Promise.all([
       buildTotalViewersPromise(),
       buildWatchTimePromise(),
       buildWatchPeakPromise(),
-      buildCountriesPromise()
+      buildGeoPromise('country'),
+      buildGeoPromise('subdivisionName')
     ])
 
     const viewersPeak = rowsWatchPeak.length !== 0
@@ -245,6 +250,11 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
       countries: rowsCountries.map(r => ({
         isoCode: r.country,
         viewers: r.viewers
+      })),
+
+      subdivisions: rowsSubdivisions.map(r => ({
+        name: r.subdivisionName,
+        viewers: r.viewers
       }))
     }
   }
@@ -347,7 +357,8 @@ export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVid
     const location = this.country
       ? {
         location: {
-          addressCountry: this.country
+          addressCountry: this.country,
+          addressRegion: this.subdivisionName
         }
       }
       : {}