From c6867725fb8e3dfbc2018a37ed5a963103587cb6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 21 Jul 2023 11:42:52 +0200 Subject: [PATCH] Add p2p info to metrics --- client/src/assets/player/peertube-player.ts | 2 +- .../shared/control-bar/p2p-info-button.ts | 20 ++++++---- .../player/shared/metrics/metrics-plugin.ts | 25 ++++-------- .../p2p-media-loader-plugin.ts | 36 ++++++++++------- .../hls-options-builder.ts | 2 + .../assets/player/shared/stats/stats-card.ts | 31 ++++++--------- .../shared/web-video/web-video-plugin.ts | 2 +- .../player/types/peertube-videojs-typings.ts | 7 +++- config/dev.yaml | 4 ++ .../metric-helpers/playback-metrics.ts | 22 +++++++++-- server/middlewares/validators/metrics.ts | 3 +- server/tests/api/check-params/metrics.ts | 11 +++++- server/tests/api/server/open-telemetry.ts | 39 ++++++++++++++++++- .../metrics/playback-metric-create.model.ts | 4 +- support/doc/api/openapi.yaml | 6 +++ 15 files changed, 141 insertions(+), 73 deletions(-) diff --git a/client/src/assets/player/peertube-player.ts b/client/src/assets/player/peertube-player.ts index ebb79247a..69ca1a566 100644 --- a/client/src/assets/player/peertube-player.ts +++ b/client/src/assets/player/peertube-player.ts @@ -203,7 +203,7 @@ export class PeerTubePlayer { this.player.one('error', () => handleError()) - this.player.on('p2p-info', (_, data: PlayerNetworkInfo) => { + this.player.on('network-info', (_, data: PlayerNetworkInfo) => { if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return saveAverageBandwidth(data.bandwidthEstimate) diff --git a/client/src/assets/player/shared/control-bar/p2p-info-button.ts b/client/src/assets/player/shared/control-bar/p2p-info-button.ts index 4177b3280..d2d2b6c99 100644 --- a/client/src/assets/player/shared/control-bar/p2p-info-button.ts +++ b/client/src/assets/player/shared/control-bar/p2p-info-button.ts @@ -39,15 +39,14 @@ class P2PInfoButton extends Button { subDivP2P.appendChild(peersText) const subDivHttp = videojs.dom.createEl('div', { className: 'vjs-peertube-hidden' }) as HTMLElement - const subDivHttpText = videojs.dom.createEl('span', { - className: 'http-fallback', - textContent: 'HTTP' - }) + const subDivHttpText = videojs.dom.createEl('span', { className: 'http-fallback' }) subDivHttp.appendChild(subDivHttpText) div.appendChild(subDivHttp) - this.player_.on('p2p-info', (_event: any, data: PlayerNetworkInfo) => { + this.player_.on('network-info', (_event: any, data: PlayerNetworkInfo) => { + if (!data.p2p) return + subDivP2P.className = 'vjs-peertube-displayed' subDivHttp.className = 'vjs-peertube-hidden' @@ -58,7 +57,7 @@ class P2PInfoButton extends Button { const uploadSpeed = bytes(p2pStats.uploadSpeed) const totalDownloaded = bytes(p2pStats.downloaded + httpStats.downloaded) const totalUploaded = bytes(p2pStats.uploaded) - const numPeers = p2pStats.numPeers + const numPeers = p2pStats.peersWithWebSeed subDivP2P.title = this.player().localize('Total downloaded: ') + totalDownloaded.join(' ') + '\n' @@ -85,8 +84,13 @@ class P2PInfoButton extends Button { subDivP2P.className = 'vjs-peertube-displayed' }) - this.player_.on('http-info', (_event, data: PlayerNetworkInfo) => { - // We are in HTTP fallback + this.player_.on('network-info', (_event, data: PlayerNetworkInfo) => { + if (data.p2p) return + + if (data.source === 'web-video') subDivHttpText.textContent = 'HTTP' + else if (data.source === 'p2p-media-loader') subDivHttpText.textContent = 'HLS' + + // We are in HTTP mode subDivHttp.className = 'vjs-peertube-displayed' subDivP2P.className = 'vjs-peertube-hidden' diff --git a/client/src/assets/player/shared/metrics/metrics-plugin.ts b/client/src/assets/player/shared/metrics/metrics-plugin.ts index 20d37d636..06ca0c2f2 100644 --- a/client/src/assets/player/shared/metrics/metrics-plugin.ts +++ b/client/src/assets/player/shared/metrics/metrics-plugin.ts @@ -19,7 +19,7 @@ class MetricsPlugin extends Plugin { private errors = 0 private p2pEnabled: boolean - private totalPeers = 0 + private p2pPeers = 0 private lastPlayerNetworkInfo: PlayerNetworkInfo @@ -111,12 +111,12 @@ class MetricsPlugin extends Plugin { errors: this.errors, - downloadedBytesP2P: this.downloadedBytesP2P, downloadedBytesHTTP: this.downloadedBytesHTTP, + downloadedBytesP2P: this.downloadedBytesP2P, uploadedBytesP2P: this.uploadedBytesP2P, - totalPeers: this.totalPeers, + p2pPeers: this.p2pPeers, p2pEnabled: this.p2pEnabled, videoId: this.options_.videoUUID() @@ -139,23 +139,14 @@ class MetricsPlugin extends Plugin { } private trackBytes () { - this.player.on('p2p-info', (_event, data: PlayerNetworkInfo) => { + this.player.on('network-info', (_event, data: PlayerNetworkInfo) => { this.downloadedBytesHTTP += Math.round(data.http.downloaded - (this.lastPlayerNetworkInfo?.http.downloaded || 0)) - this.downloadedBytesP2P += Math.round(data.p2p.downloaded - (this.lastPlayerNetworkInfo?.p2p.downloaded || 0)) + this.downloadedBytesP2P += Math.round((data.p2p?.downloaded || 0) - (this.lastPlayerNetworkInfo?.p2p?.downloaded || 0)) - this.uploadedBytesP2P += Math.round(data.p2p.uploaded - (this.lastPlayerNetworkInfo?.p2p.uploaded || 0)) + this.uploadedBytesP2P += Math.round((data.p2p?.uploaded || 0) - (this.lastPlayerNetworkInfo?.p2p?.uploaded || 0)) - this.totalPeers = data.p2p.numPeers - this.p2pEnabled = true - - this.lastPlayerNetworkInfo = data - }) - - this.player.on('http-info', (_event, data: PlayerNetworkInfo) => { - this.downloadedBytesHTTP += Math.round(data.http.downloaded - (this.lastPlayerNetworkInfo?.http.downloaded || 0)) - - this.totalPeers = 0 - this.p2pEnabled = false + this.p2pPeers = data.p2p?.peersP2POnly + this.p2pEnabled = !!data.p2p this.lastPlayerNetworkInfo = data }) diff --git a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts index fe967a730..8c376cd21 100644 --- a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts +++ b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts @@ -16,7 +16,8 @@ class P2pMediaLoaderPlugin extends Plugin { private statsP2PBytes = { pendingDownload: [] as number[], pendingUpload: [] as number[], - numPeers: 0, + peersWithWebSeed: 0, + peersP2POnly: 0, totalDownload: 0, totalUpload: 0 } @@ -113,7 +114,7 @@ class P2pMediaLoaderPlugin extends Plugin { this.options.redundancyUrlManager.removeBySegmentUrl(segment.requestUrl) }) - this.statsP2PBytes.numPeers = 1 + this.options.redundancyUrlManager.countBaseUrls() + this.statsP2PBytes.peersWithWebSeed = 1 + this.options.redundancyUrlManager.countBaseUrls() this.runStats() @@ -138,8 +139,14 @@ class P2pMediaLoaderPlugin extends Plugin { this.statsP2PBytes.totalUpload += bytes }) - this.p2pEngine.on(Events.PeerConnect, () => this.statsP2PBytes.numPeers++) - this.p2pEngine.on(Events.PeerClose, () => this.statsP2PBytes.numPeers--) + this.p2pEngine.on(Events.PeerConnect, () => { + this.statsP2PBytes.peersWithWebSeed++ + this.statsP2PBytes.peersP2POnly++ + }) + this.p2pEngine.on(Events.PeerClose, () => { + this.statsP2PBytes.peersWithWebSeed-- + this.statsP2PBytes.peersP2POnly-- + }) this.networkInfoInterval = setInterval(() => { const p2pDownloadSpeed = this.arraySum(this.statsP2PBytes.pendingDownload) @@ -151,20 +158,23 @@ class P2pMediaLoaderPlugin extends Plugin { this.statsP2PBytes.pendingUpload = [] this.statsHTTPBytes.pendingDownload = [] - return this.player.trigger('p2p-info', { + return this.player.trigger('network-info', { source: 'p2p-media-loader', + bandwidthEstimate: (this.hlsjs as any).bandwidthEstimate / 8, http: { downloadSpeed: httpDownloadSpeed, downloaded: this.statsHTTPBytes.totalDownload }, - p2p: { - downloadSpeed: p2pDownloadSpeed, - uploadSpeed: p2pUploadSpeed, - numPeers: this.statsP2PBytes.numPeers, - downloaded: this.statsP2PBytes.totalDownload, - uploaded: this.statsP2PBytes.totalUpload - }, - bandwidthEstimate: (this.hlsjs as any).bandwidthEstimate / 8 + p2p: this.options.p2pEnabled + ? { + downloadSpeed: p2pDownloadSpeed, + uploadSpeed: p2pUploadSpeed, + peersWithWebSeed: this.statsP2PBytes.peersWithWebSeed, + peersP2POnly: this.statsP2PBytes.peersP2POnly, + downloaded: this.statsP2PBytes.totalDownload, + uploaded: this.statsP2PBytes.totalUpload + } + : undefined } as PlayerNetworkInfo) }, 1000) } diff --git a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts index 1658f776c..fd632d90d 100644 --- a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts +++ b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts @@ -44,6 +44,8 @@ export class HLSOptionsBuilder { requiresUserAuth: this.options.requiresUserAuth, videoFileToken: this.options.videoFileToken, + p2pEnabled: this.options.p2pEnabled, + redundancyUrlManager, type: 'application/x-mpegURL', src: this.options.hls.playlistUrl, diff --git a/client/src/assets/player/shared/stats/stats-card.ts b/client/src/assets/player/shared/stats/stats-card.ts index 077c900e5..13334d91a 100644 --- a/client/src/assets/player/shared/stats/stats-card.ts +++ b/client/src/assets/player/shared/stats/stats-card.ts @@ -63,8 +63,7 @@ class StatsCard extends Component { private liveLatency: InfoElement - private onP2PInfoHandler: (_event: any, data: EventPlayerNetworkInfo) => void - private onHTTPInfoHandler: (_event: any, data: EventPlayerNetworkInfo) => void + private onNetworkInfoHandler: (_event: any, data: EventPlayerNetworkInfo) => void createEl () { this.containerEl = videojs.dom.createEl('div', { @@ -89,33 +88,26 @@ class StatsCard extends Component { this.populateInfoBlocks() - this.onP2PInfoHandler = (_event, data) => { + this.onNetworkInfoHandler = (_event, data) => { this.mode = data.source const p2pStats = data.p2p const httpStats = data.http - this.playerNetworkInfo.downloadSpeed = bytes(p2pStats.downloadSpeed + httpStats.downloadSpeed).join(' ') - this.playerNetworkInfo.uploadSpeed = bytes(p2pStats.uploadSpeed).join(' ') - this.playerNetworkInfo.totalDownloaded = bytes(p2pStats.downloaded + httpStats.downloaded).join(' ') - this.playerNetworkInfo.totalUploaded = bytes(p2pStats.uploaded).join(' ') - this.playerNetworkInfo.numPeers = p2pStats.numPeers - this.playerNetworkInfo.averageBandwidth = bytes(data.bandwidthEstimate).join(' ') + '/s' + this.playerNetworkInfo.downloadSpeed = bytes((p2pStats?.downloadSpeed || 0) + (httpStats.downloadSpeed || 0)).join(' ') + this.playerNetworkInfo.uploadSpeed = bytes(p2pStats?.uploadSpeed || 0).join(' ') + this.playerNetworkInfo.totalDownloaded = bytes((p2pStats?.downloaded || 0) + httpStats.downloaded).join(' ') + this.playerNetworkInfo.totalUploaded = bytes(p2pStats?.uploaded || 0).join(' ') + this.playerNetworkInfo.numPeers = p2pStats?.peersWithWebSeed if (data.source === 'p2p-media-loader') { + this.playerNetworkInfo.averageBandwidth = bytes(data.bandwidthEstimate).join(' ') + '/s' this.playerNetworkInfo.downloadedFromServer = bytes(httpStats.downloaded).join(' ') - this.playerNetworkInfo.downloadedFromPeers = bytes(p2pStats.downloaded).join(' ') + this.playerNetworkInfo.downloadedFromPeers = bytes(p2pStats?.downloaded || 0).join(' ') } } - this.onHTTPInfoHandler = (_event, data) => { - this.mode = data.source - - this.playerNetworkInfo.totalDownloaded = bytes(data.http.downloaded).join(' ') - } - - this.player().on('p2p-info', this.onP2PInfoHandler) - this.player().on('http-info', this.onHTTPInfoHandler) + this.player().on('network-info', this.onNetworkInfoHandler) return this.containerEl } @@ -123,8 +115,7 @@ class StatsCard extends Component { dispose () { if (this.updateInterval) clearInterval(this.updateInterval) - this.player().off('p2p-info', this.onP2PInfoHandler) - this.player().off('http-info', this.onHTTPInfoHandler) + this.player().off('network-info', this.onNetworkInfoHandler) super.dispose() } diff --git a/client/src/assets/player/shared/web-video/web-video-plugin.ts b/client/src/assets/player/shared/web-video/web-video-plugin.ts index 930b5045a..d09b5a724 100644 --- a/client/src/assets/player/shared/web-video/web-video-plugin.ts +++ b/client/src/assets/player/shared/web-video/web-video-plugin.ts @@ -175,7 +175,7 @@ class WebVideoPlugin extends Plugin { private setupNetworkInfoInterval () { this.networkInfoInterval = setInterval(() => { - return this.player.trigger('http-info', { + return this.player.trigger('network-info', { source: 'web-video', http: { downloaded: this.player.bufferedPercent() * this.currentVideoFile.size diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts index adaa43d77..73b38d0d3 100644 --- a/client/src/assets/player/types/peertube-videojs-typings.ts +++ b/client/src/assets/player/types/peertube-videojs-typings.ts @@ -184,6 +184,8 @@ type P2PMediaLoaderPluginOptions = { type: string src: string + p2pEnabled: boolean + loader: P2PMediaLoader segmentValidator: SegmentValidator @@ -240,9 +242,12 @@ type PlayerNetworkInfo = { p2p?: { downloadSpeed: number uploadSpeed: number + downloaded: number uploaded: number - numPeers: number + + peersWithWebSeed: number + peersP2POnly: number } // In bytes diff --git a/config/dev.yaml b/config/dev.yaml index f570ede1e..60179533d 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -35,6 +35,10 @@ smtp: log: level: 'debug' +open_telemetry: + metrics: + enabled: true + contact_form: enabled: true diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts index 41a5dd640..1eb08b5a6 100644 --- a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts +++ b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts @@ -1,4 +1,4 @@ -import { Counter, Histogram, Meter } from '@opentelemetry/api' +import { Counter, Meter } from '@opentelemetry/api' import { MVideoImmutable } from '@server/types/models' import { PlaybackMetricCreate } from '@shared/models' @@ -11,7 +11,10 @@ export class PlaybackMetrics { private downloadedBytesHTTPCounter: Counter - private peersP2PPeers: Histogram + private peersP2PPeersGaugeBuffer: { + value: number + attributes: any + }[] = [] constructor (private readonly meter: Meter) { @@ -37,8 +40,14 @@ export class PlaybackMetrics { description: 'Uploaded bytes with P2P by PeerTube player.' }) - this.peersP2PPeers = this.meter.createHistogram('peertube_playback_p2p_peers', { + this.meter.createObservableGauge('peertube_playback_p2p_peers', { description: 'Total P2P peers connected to the PeerTube player.' + }).addCallback(observableResult => { + for (const gauge of this.peersP2PPeersGaugeBuffer) { + observableResult.observe(gauge.value, gauge.attributes) + } + + this.peersP2PPeersGaugeBuffer = [] }) } @@ -66,6 +75,11 @@ export class PlaybackMetrics { this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) - if (metrics.totalPeers) this.peersP2PPeers.record(metrics.totalPeers, attributes) + if (metrics.p2pPeers) { + this.peersP2PPeersGaugeBuffer.push({ + value: metrics.p2pPeers, + attributes + }) + } } } diff --git a/server/middlewares/validators/metrics.ts b/server/middlewares/validators/metrics.ts index ced9afc69..986b30a19 100644 --- a/server/middlewares/validators/metrics.ts +++ b/server/middlewares/validators/metrics.ts @@ -13,12 +13,11 @@ const addPlaybackMetricValidator = [ .optional() .isInt({ min: 0 }), - body('totalPeers') + body('p2pPeers') .optional() .isInt({ min: 0 }), body('p2pEnabled') - .optional() .isBoolean(), body('playerMode') diff --git a/server/tests/api/check-params/metrics.ts b/server/tests/api/check-params/metrics.ts index 25443401d..302bef4f5 100644 --- a/server/tests/api/check-params/metrics.ts +++ b/server/tests/api/check-params/metrics.ts @@ -38,6 +38,7 @@ describe('Test metrics API validators', function () { fps: 30, resolutionChanges: 1, errors: 2, + p2pEnabled: true, downloadedBytesP2P: 0, downloadedBytesHTTP: 0, uploadedBytesP2P: 0, @@ -145,7 +146,13 @@ describe('Test metrics API validators', function () { }) }) - it('Should fail with an invalid p2pEnabled', async function () { + it('Should fail with a missing/invalid p2pEnabled', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'p2pEnabled' ]) + }) + await makePostBodyRequest({ url: server.url, path, @@ -157,7 +164,7 @@ describe('Test metrics API validators', function () { await makePostBodyRequest({ url: server.url, path, - fields: { ...baseParams, totalPeers: 'toto' } + fields: { ...baseParams, p2pPeers: 'toto' } }) }) diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts index 0bd0b5e86..508e9d649 100644 --- a/server/tests/api/server/open-telemetry.ts +++ b/server/tests/api/server/open-telemetry.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared' -import { HttpStatusCode, VideoPrivacy, VideoResolution } from '@shared/models' +import { HttpStatusCode, PlaybackMetricCreate, VideoPrivacy, VideoResolution } from '@shared/models' import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' describe('Open Telemetry', function () { @@ -62,14 +62,49 @@ describe('Open Telemetry', function () { downloadedBytesP2P: 0, downloadedBytesHTTP: 0, uploadedBytesP2P: 5, - totalPeers: 1, + p2pPeers: 1, p2pEnabled: false, videoId: video.uuid } }) const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') + expect(res.text).to.contain('peertube_playback_p2p_peers{') + expect(res.text).to.contain('p2pEnabled="false"') + }) + + it('Should take the last playback metric', async function () { + await setAccessTokensToServers([ server ]) + + const video = await server.videos.quickUpload({ name: 'video' }) + + const metrics = { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 5, + p2pPeers: 7, + p2pEnabled: false, + videoId: video.uuid + } as PlaybackMetricCreate + + await server.metrics.addPlaybackMetric({ metrics }) + + metrics.p2pPeers = 42 + await server.metrics.addPlaybackMetric({ metrics }) + + const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + + // eslint-disable-next-line max-len + const label = `{videoOrigin="local",playerMode="p2p-media-loader",resolution="1080",fps="30",p2pEnabled="false",videoUUID="${video.uuid}"}` + expect(res.text).to.contain(`peertube_playback_p2p_peers${label} 42`) + expect(res.text).to.not.contain(`peertube_playback_p2p_peers${label} 7`) }) it('Should disable http request duration metrics', async function () { diff --git a/shared/models/metrics/playback-metric-create.model.ts b/shared/models/metrics/playback-metric-create.model.ts index b428beeb6..1d47421c3 100644 --- a/shared/models/metrics/playback-metric-create.model.ts +++ b/shared/models/metrics/playback-metric-create.model.ts @@ -6,8 +6,8 @@ export interface PlaybackMetricCreate { resolution?: VideoResolution fps?: number - p2pEnabled?: boolean - totalPeers?: number + p2pEnabled: boolean + p2pPeers?: number resolutionChanges: number diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 2dfad9987..90aaebd26 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -9528,6 +9528,11 @@ components: fps: type: number description: Current player video fps + p2pEnabled: + type: boolean + p2pPeers: + type: number + description: P2P peers connected (doesn't include WebSeed peers) resolutionChanges: type: number description: How many resolution changes occured since the last metric creation @@ -9555,6 +9560,7 @@ components: - downloadedBytesP2P - downloadedBytesHTTP - uploadedBytesP2P + - p2pEnabled - videoId RunnerRegistrationToken: