Anonymize IP stored in Redis too

scripts/simulate-many-viewers-worker.js

@@ -0,0 +1,9 @@
+import Bluebird from "bluebird";
+import { PeerTubeServer } from "@peertube/peertube-server-commands";
+module.exports = async function sendViews(options) {
+  const { url, videoId, viewers } = options;
+  const server = new PeerTubeServer({ url });
+  await Bluebird.map(viewers, (viewer) => {
+    return server.views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor }).catch((err) => console.error("Cannot simulate viewer", err));
+  }, { concurrency: 500 });
+};

scripts/simulate-many-viewers-worker.ts

@@ -0,0 +1,17 @@
+import Bluebird from 'bluebird'
+import { PeerTubeServer } from '@peertube/peertube-server-commands'
+
+module.exports = async function sendViews (options: {
+  url: string
+  videoId: number
+  viewers: { xForwardedFor: string }[]
+}) {
+  const { url, videoId, viewers } = options
+
+  const server = new PeerTubeServer({ url })
+
+  await Bluebird.map(viewers, viewer => {
+    return server.views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor })
+      .catch(err => console.error('Cannot simulate viewer', err))
+  }, { concurrency: 500 })
+}

scripts/simulate-many-viewers.ts

@@ -1,4 +1,3 @@
-import Bluebird from 'bluebird'
 import { wait } from '@peertube/peertube-core-utils'
 import {
   createSingleServer,
@@ -8,15 +7,27 @@ import {
   setAccessTokensToServers,
   waitJobs
 } from '@peertube/peertube-server-commands'
+import { isMainThread } from 'worker_threads'
+import Piscina from 'piscina'
+import { fileURLToPath } from 'url'
+import { dirname, join } from 'path'
+
+const THOUSAND_VIEWERS = 2
+const TOTAL_THREADS = 20
 
 let servers: PeerTubeServer[]
 const viewers: { xForwardedFor: string }[] = []
 let videoId: string
+let pool: Piscina
 
-run()
+if (isMainThread) {
+  run()
     .then(() => process.exit(0))
     .catch(err => console.error(err))
     .finally(() => killallServers(servers))
+}
+
+// ---------------------------------------------------------------------------
 
 async function run () {
   await prepare()
@@ -27,6 +38,12 @@ async function run () {
 }
 
 async function prepare () {
+  pool = new Piscina({
+    filename: join(dirname(fileURLToPath(import.meta.url)), 'simulate-many-viewers-worker.js'),
+    minThreads: 20,
+    maxThreads: 20
+  })
+
   console.log('Preparing servers...')
 
   const config = {
@@ -64,8 +81,6 @@ async function prepare () {
 
   await waitJobs(servers)
 
-  const THOUSAND_VIEWERS = 2
-
   for (let i = 2; i < 252; i++) {
     for (let j = 2; j < 6; j++) {
       for (let k = 2; k < THOUSAND_VIEWERS + 2; k++) {
@@ -82,9 +97,18 @@ async function runViewers () {
 
   const before = new Date().getTime()
 
-  await Bluebird.map(viewers, viewer => {
+  const promises: Promise<any>[] = []
-    return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor })
+
-  }, { concurrency: 500 })
+  for (let i = 0; i < TOTAL_THREADS; i++) {
+    const start = i * THOUSAND_VIEWERS * 1000 / TOTAL_THREADS
+    const end = (i + 1) * THOUSAND_VIEWERS * 1000 / TOTAL_THREADS
+
+    console.log(`Sending viewers ${start} to ${end}`)
+
+    promises.push(pool.run({ url: servers[0].url, viewers: viewers.slice(start, end), videoId }))
+  }
+
+  await Promise.all(promises)
 
   console.log('Finished to run views in %d seconds.', (new Date().getTime() - before) / 1000)
 

server/core/lib/redis.ts

@@ -341,7 +341,9 @@ class Redis {
   generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string }
   generateLocalVideoViewerKeys (): { setKey: string }
   generateLocalVideoViewerKeys (ip?: string, videoId?: number) {
-    return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` }
+    const anonymousIP = sha256(CONFIG.SECRETS + '-' + ip)
+
+    return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${anonymousIP}-${videoId}` }
   }
 
   private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) {
@@ -369,11 +371,11 @@ class Redis {
   }
 
   generateIPViewKey (ip: string, videoUUID: string) {
-    return `views-${videoUUID}-${ip}`
+    return `views-${videoUUID}-${sha256(CONFIG.SECRETS.PEERTUBE + '-' + ip)}`
   }
 
   private generateContactFormKey (ip: string) {
-    return 'contact-form-' + ip
+    return 'contact-form-' + sha256(CONFIG.SECRETS.PEERTUBE + '-' + ip)
   }
 
   private generateAPUnavailabilityKey (url: string) {