Add logs page in client

client/src/app/+admin/admin-routing.module.ts

@@ -6,9 +6,9 @@ import { MetaGuard } from '@ngx-meta/core'
 
 import { AdminComponent } from './admin.component'
 import { FollowsRoutes } from './follows'
-import { JobsRoutes } from './jobs/job.routes'
 import { UsersRoutes } from './users'
 import { ModerationRoutes } from '@app/+admin/moderation/moderation.routes'
+import { SystemRoutes } from '@app/+admin/system'
 
 const adminRoutes: Routes = [
   {
@@ -25,7 +25,7 @@ const adminRoutes: Routes = [
       ...FollowsRoutes,
       ...UsersRoutes,
       ...ModerationRoutes,
-      ...JobsRoutes,
+      ...SystemRoutes,
       ...ConfigRoutes
     ]
   }

client/src/app/+admin/admin.component.html

@@ -12,13 +12,13 @@
       Moderation
     </a>
 
-    <a i18n *ngIf="hasJobsRight()" routerLink="/admin/jobs" routerLinkActive="active" class="title-page">
-      Jobs
-    </a>
-
     <a i18n *ngIf="hasConfigRight()" routerLink="/admin/config" routerLinkActive="active" class="title-page">
       Configuration
     </a>
+
+    <a i18n *ngIf="hasJobsRight() || hasLogsRight()" routerLink="/admin/system" routerLinkActive="active" class="title-page">
+      System
+    </a>
   </div>
 
   <div class="margin-content">

client/src/app/+admin/admin.component.ts

@@ -28,6 +28,10 @@ export class AdminComponent {
     return this.auth.getUser().hasRight(UserRight.MANAGE_JOBS)
   }
 
+  hasLogsRight () {
+    return this.auth.getUser().hasRight(UserRight.MANAGE_LOGS)
+  }
+
   hasConfigRight () {
     return this.auth.getUser().hasRight(UserRight.MANAGE_CONFIGURATION)
   }

client/src/app/+admin/admin.module.ts

@@ -7,20 +7,19 @@ import { AdminRoutingModule } from './admin-routing.module'
 import { AdminComponent } from './admin.component'
 import { FollowersListComponent, FollowingAddComponent, FollowsComponent, FollowService } from './follows'
 import { FollowingListComponent } from './follows/following-list/following-list.component'
-import { JobsComponent } from './jobs/job.component'
+import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users'
-import { JobsListComponent } from './jobs/jobs-list/jobs-list.component'
-import { JobService } from './jobs/shared/job.service'
-import { UserCreateComponent, UserListComponent, UsersComponent, UserUpdateComponent, UserPasswordComponent } from './users'
 import {
   ModerationCommentModalComponent,
   VideoAbuseListComponent,
-  VideoBlacklistListComponent,
+  VideoAutoBlacklistListComponent,
-  VideoAutoBlacklistListComponent
+  VideoBlacklistListComponent
 } from './moderation'
 import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
 import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
 import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
 import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist'
+import { JobsComponent } from '@app/+admin/system/jobs/jobs.component'
+import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system'
 
 @NgModule({
   imports: [
@@ -52,8 +51,9 @@ import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } f
     InstanceServerBlocklistComponent,
     InstanceAccountBlocklistComponent,
 
+    SystemComponent,
     JobsComponent,
-    JobsListComponent,
+    LogsComponent,
 
     ConfigComponent,
     EditCustomConfigComponent
@@ -67,6 +67,7 @@ import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } f
     FollowService,
     RedundancyService,
     JobService,
+    LogsService,
     ConfigService
   ]
 })

client/src/app/+admin/system/index.ts

@@ -0,0 +1,4 @@
+export * from './jobs'
+export * from './logs'
+export * from './system.component'
+export * from './system.routes'

client/src/app/+admin/system/jobs/index.ts

@@ -1,4 +1,2 @@
-export * from './shared'
+export * from './job.service'
-export * from './jobs-list'
+export * from './jobs.component'
-export * from './job.routes'
-export * from './job.component'

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -5,15 +5,15 @@ import { SortMeta } from 'primeng/primeng'
 import { Job } from '../../../../../../shared/index'
 import { JobState } from '../../../../../../shared/models'
 import { RestPagination, RestTable } from '../../../shared'
-import { JobService } from '../shared'
+import { JobService } from './job.service'
 import { I18n } from '@ngx-translate/i18n-polyfill'
 
 @Component({
-  selector: 'my-jobs-list',
+  selector: 'my-jobs',
-  templateUrl: './jobs-list.component.html',
+  templateUrl: './jobs.component.html',
-  styleUrls: [ './jobs-list.component.scss' ]
+  styleUrls: [ './jobs.component.scss' ]
 })
-export class JobsListComponent extends RestTable implements OnInit {
+export class JobsComponent extends RestTable implements OnInit {
   private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state'
 
   jobState: JobState = 'waiting'
@@ -58,12 +58,12 @@ export class JobsListComponent extends RestTable implements OnInit {
   }
 
   private loadJobState () {
-    const result = peertubeLocalStorage.getItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE)
+    const result = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE)
 
     if (result) this.jobState = result as JobState
   }
 
   private saveJobState () {
-    peertubeLocalStorage.setItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState)
+    peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState)
   }
 }

client/src/app/+admin/system/logs/index.ts

@@ -0,0 +1,2 @@
+export * from './logs.component'
+export * from './logs.service'

client/src/app/+admin/system/logs/log-row.model.ts

@@ -0,0 +1,21 @@
+import { LogLevel } from '@shared/models/server/log-level.type'
+import omit from 'lodash-es/omit'
+
+export class LogRow {
+  date: Date
+  localeDate: string
+  level: LogLevel
+  message: string
+  meta: string
+
+  constructor (row: any) {
+    this.date = new Date(row.timestamp)
+    this.localeDate = this.date.toLocaleString()
+    this.level = row.level
+    this.message = row.message
+
+    const metaObj = omit(row, 'timestamp', 'level', 'message', 'label')
+
+    if (Object.keys(metaObj).length !== 0) this.meta = JSON.stringify(metaObj, undefined, 2)
+  }
+}

client/src/app/+admin/system/logs/logs.component.html

@@ -0,0 +1,31 @@
+<div class="header">
+  <div class="peertube-select-container">
+    <select [(ngModel)]="startDate" (ngModelChange)="refresh()">
+      <option *ngFor="let timeChoice of timeChoices" [value]="timeChoice.id">{{ timeChoice.label }}</option>
+    </select>
+  </div>
+
+  <div class="peertube-select-container">
+    <select [(ngModel)]="level" (ngModelChange)="refresh()">
+      <option *ngFor="let levelChoice of levelChoices" [value]="levelChoice.id">{{ levelChoice.label }}</option>
+    </select>
+  </div>
+
+  <my-button i18n-label label="Refresh" icon="refresh" (click)="refresh()"></my-button>
+</div>
+
+<div class="logs">
+  <div *ngIf="loading">Loading...</div>
+
+  <div #logsElement>
+    <div *ngFor="let log of logs" class="log-row" [ngClass]="{ error: log.level === 'error', warn: log.level === 'warn' }">
+      <span class="log-level">{{ log.level }}</span>
+
+      <span class="log-date">[{{ log.localeDate }}]</span>
+
+      {{ log.message }}
+
+      {{ log.meta }}
+    </div>
+  </div>
+</div>

client/src/app/+admin/system/logs/logs.component.scss

@@ -0,0 +1,48 @@
+@import '_variables';
+@import '_mixins';
+
+.logs {
+  font-family: monospace;
+  font-size: 13px;
+  max-height: 500px;
+  overflow-y: auto;
+  background: rgba(0, 0, 0, 0.03);
+  padding: 20px;
+
+  .log-row {
+    margin-top: 1px;
+
+    &:hover {
+      background: rgba(0, 0, 0, 0.07);
+    }
+  }
+
+  .log-level {
+    font-weight: $font-semibold;
+    margin-right: 5px;
+  }
+
+  .warn {
+    color: $orange-color;
+  }
+
+  .error {
+    color: $red;
+  }
+}
+
+.header {
+  display: flex;
+  justify-content: flex-end;
+  margin-bottom: 10px;
+
+  .peertube-select-container {
+    @include peertube-select-container(150px);
+  }
+
+  my-button,
+  .peertube-select-container {
+    margin-left: 10px;
+  }
+}
+

client/src/app/+admin/system/logs/logs.component.ts

@@ -0,0 +1,111 @@
+import { Component, ElementRef, OnInit, ViewChild } from '@angular/core'
+import { LogsService } from '@app/+admin/system/logs/logs.service'
+import { Notifier } from '@app/core'
+import { LogRow } from '@app/+admin/system/logs/log-row.model'
+import { I18n } from '@ngx-translate/i18n-polyfill'
+import { LogLevel } from '@shared/models/server/log-level.type'
+
+@Component({
+  templateUrl: './logs.component.html',
+  styleUrls: [ './logs.component.scss' ]
+})
+export class LogsComponent implements OnInit {
+  @ViewChild('logsElement') logsElement: ElementRef<HTMLElement>
+
+  loading = false
+
+  logs: LogRow[] = []
+  timeChoices: { id: string, label: string }[] = []
+  levelChoices: { id: LogLevel, label: string }[] = []
+
+  startDate: string
+  level: LogLevel
+
+  constructor (
+    private logsService: LogsService,
+    private notifier: Notifier,
+    private i18n: I18n
+  ) { }
+
+  ngOnInit (): void {
+    this.buildTimeChoices()
+    this.buildLevelChoices()
+
+    this.load()
+  }
+
+  refresh () {
+    this.logs = []
+    this.load()
+  }
+
+  load () {
+    this.loading = true
+
+    this.logsService.getLogs(this.level, this.startDate)
+        .subscribe(
+          logs => {
+            this.logs = logs
+
+            setTimeout(() => {
+              this.logsElement.nativeElement.scrollIntoView({ block: 'end', inline: 'nearest' })
+            })
+          },
+
+          err => this.notifier.error(err.message),
+
+          () => this.loading = false
+        )
+  }
+
+  buildTimeChoices () {
+    const lastHour = new Date()
+    lastHour.setHours(lastHour.getHours() - 1)
+
+    const lastDay = new Date()
+    lastDay.setDate(lastDay.getDate() - 1)
+
+    const lastWeek = new Date()
+    lastWeek.setDate(lastWeek.getDate() - 7)
+
+    this.timeChoices = [
+      {
+        id: lastWeek.toISOString(),
+        label: this.i18n('Last week')
+      },
+      {
+        id: lastDay.toISOString(),
+        label: this.i18n('Last day')
+      },
+      {
+        id: lastHour.toISOString(),
+        label: this.i18n('Last hour')
+      }
+    ]
+
+    this.startDate = lastHour.toISOString()
+  }
+
+  buildLevelChoices () {
+    this.levelChoices = [
+      {
+        id: 'debug',
+        label: this.i18n('Debug')
+      },
+      {
+        id: 'info',
+        label: this.i18n('Info')
+      },
+      {
+        id: 'warn',
+        label: this.i18n('Warning')
+      },
+      {
+        id: 'error',
+        label: this.i18n('Error')
+      }
+    ]
+
+    this.level = 'info'
+  }
+}

client/src/app/+admin/system/logs/logs.service.ts

@@ -0,0 +1,33 @@
+import { catchError, map } from 'rxjs/operators'
+import { HttpClient, HttpParams } from '@angular/common/http'
+import { Injectable } from '@angular/core'
+import { Observable } from 'rxjs'
+import { environment } from '../../../../environments/environment'
+import { RestExtractor, RestService } from '../../../shared'
+import { LogRow } from '@app/+admin/system/logs/log-row.model'
+import { LogLevel } from '@shared/models/server/log-level.type'
+
+@Injectable()
+export class LogsService {
+  private static BASE_JOB_URL = environment.apiUrl + '/api/v1/server/logs'
+
+  constructor (
+    private authHttp: HttpClient,
+    private restService: RestService,
+    private restExtractor: RestExtractor
+  ) {}
+
+  getLogs (level: LogLevel, startDate: string, endDate?: string): Observable<any> {
+    let params = new HttpParams()
+    params = params.append('startDate', startDate)
+    params = params.append('level', level)
+
+    if (endDate) params.append('endDate', endDate)
+
+    return this.authHttp.get<any[]>(LogsService.BASE_JOB_URL, { params })
+               .pipe(
+                 map(rows => rows.map(r => new LogRow(r))),
+                 catchError(err => this.restExtractor.handleError(err))
+               )
+  }
+}

client/src/app/+admin/system/system.component.html

@@ -0,0 +1,11 @@
+<div class="admin-sub-header">
+  <div i18n class="form-sub-title">System</div>
+
+  <div class="admin-sub-nav">
+    <a i18n routerLink="jobs" routerLinkActive="active">Jobs</a>
+
+    <a i18n routerLink="logs" routerLinkActive="active">Logs</a>
+  </div>
+</div>
+
+<router-outlet></router-outlet>

client/src/app/+admin/system/system.component.scss

@@ -0,0 +1,4 @@
+.form-sub-title {
+  flex-grow: 0;
+  margin-right: 30px;
+}

client/src/app/+admin/system/system.component.ts

@@ -0,0 +1,8 @@
+import { Component } from '@angular/core'
+
+@Component({
+  templateUrl: './system.component.html',
+  styleUrls: [ './system.component.scss' ]
+})
+export class SystemComponent {
+}

client/src/app/+admin/system/system.routes.ts

@@ -0,0 +1,44 @@
+import { Routes } from '@angular/router'
+import { UserRightGuard } from '../../core'
+import { UserRight } from '../../../../../shared'
+import { JobsComponent } from '@app/+admin/system/jobs/jobs.component'
+import { LogsComponent } from '@app/+admin/system/logs'
+import { SystemComponent } from '@app/+admin/system/system.component'
+
+export const SystemRoutes: Routes = [
+  {
+    path: 'system',
+    component: SystemComponent,
+    data: {
+    },
+    children: [
+      {
+        path: '',
+        redirectTo: 'jobs',
+        pathMatch: 'full'
+      },
+      {
+        path: 'jobs',
+        canActivate: [ UserRightGuard ],
+        component: JobsComponent,
+        data: {
+          meta: {
+            userRight: UserRight.MANAGE_JOBS,
+            title: 'Jobs'
+          }
+        }
+      },
+      {
+        path: 'logs',
+        canActivate: [ UserRightGuard ],
+        component: LogsComponent,
+        data: {
+          meta: {
+            userRight: UserRight.MANAGE_LOGS,
+            title: 'Logs'
+          }
+        }
+      }
+    ]
+  }
+]

client/src/app/shared/images/global-icon.component.ts

@@ -44,7 +44,8 @@ const icons = {
   'folder': require('../../../assets/images/global/folder.html'),
   'administration': require('../../../assets/images/menu/administration.html'),
   'subscriptions': require('../../../assets/images/menu/subscriptions.html'),
-  'users': require('../../../assets/images/global/users.html')
+  'users': require('../../../assets/images/global/users.html'),
+  'refresh': require('../../../assets/images/global/refresh.html')
 }
 
 export type GlobalIconName = keyof typeof icons

client/src/assets/images/global/refresh.html

@@ -0,0 +1,12 @@
+<svg width="24px" height="24px" viewBox="0 0 24 24" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+  <defs/>
+  <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+    <g id="Artboard-4" transform="translate(-224.000000, -1046.000000)" fill="#000000">
+      <g id="Extras" transform="translate(48.000000, 1046.000000)">
+        <g id="refresh" transform="translate(176.000000, 0.000000)">
+          <path d="M20.9995201,13.0312796 L20.9999519,13.0312796 C20.9830843,17.9874565 16.960132,22 12,22 C7.02943725,22 3,17.9705627 3,13 C3,8.0398348 7.01259713,4.01686187 11.9688198,4.00005287 L11.9688198,6.00006796 C8.11716976,6.01686496 5,9.14440548 5,13 C5,16.8659932 8.13400675,20 12,20 C15.8555614,20 18.9830812,16.8828839 18.9999316,13.0312796 L19.0004799,13.0312796 C19.0001607,13.0208922 19,13.0104649 19,13 C19,12.4477153 19.4477153,12 20,12 C20.5522847,12 21,12.4477153 21,13 C21,13.0104649 20.9998393,13.0208922 20.9995201,13.0312796 Z M12,9 L12,1 L16,5 L12,9 Z" id="Combined-Shape"/>
+        </g>
+      </g>
+    </g>
+  </g>
+</svg>

server/controllers/api/server/logs.ts

@@ -2,10 +2,8 @@ import * as express from 'express'
 import { UserRight } from '../../../../shared/models/users'
 import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
 import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs'
-import { readdir } from 'fs-extra'
+import { readdir, readFile } from 'fs-extra'
 import { CONFIG, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers'
-import { createInterface } from 'readline'
-import { createReadStream } from 'fs'
 import { join } from 'path'
 import { getLogsValidator } from '../../../middlewares/validators/logs'
 import { LogLevel } from '../../../../shared/models/server/log-level.type'
@@ -36,7 +34,7 @@ async function getLogs (req: express.Request, res: express.Response) {
   const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date()
   const level: LogLevel = req.query.level || 'info'
 
-  let output = ''
+  let output: string[] = []
 
   for (const meta of sortedLogFiles) {
     const path = join(CONFIG.STORAGE.LOG_DIR, meta.file)
@@ -44,18 +42,19 @@ async function getLogs (req: express.Request, res: express.Response) {
     const result = await getOutputFromFile(path, startDate, endDate, level, currentSize)
     if (!result.output) break
 
-    output = output + result.output
+    output = result.output.concat(output)
     currentSize = result.currentSize
 
-    if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break
+    if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break
   }
 
   return res.json(output).end()
 }
 
-function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) {
+async function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) {
   const startTime = startDate.getTime()
   const endTime = endDate.getTime()
+  let logTime: number
 
   const logsLevel: { [ id in LogLevel ]: number } = {
     debug: 0,
@@ -64,27 +63,32 @@ function getOutputFromFile (path: string, startDate: Date, endDate: Date, level:
     error: 3
   }
 
-  return new Promise<{ output: string, currentSize: number }>(res => {
+  const content = await readFile(path)
-    const stream = createReadStream(path)
+  const lines = content.toString().split('\n')
-    let output = ''
+  const output: any[] = []
 
-    stream.once('close', () => res({ output, currentSize }))
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const line = lines[ i ]
+    let log: any
 
-    const rl = createInterface({
+    try {
-      input: stream
+      log = JSON.parse(line)
-    })
+    } catch {
+      // Maybe there a multiple \n at the end of the file
+      continue
+    }
 
-    rl.on('line', line => {
+    logTime = new Date(log.timestamp).getTime()
-      const log = JSON.parse(line)
+    if (logTime >= startTime && logTime <= endTime && logsLevel[ log.level ] >= logsLevel[ level ]) {
+      output.push(log)
 
-      const logTime = new Date(log.timestamp).getTime()
+      currentSize += line.length
-      if (logTime >= startTime && logTime <= endTime && logsLevel[log.level] >= logsLevel[level]) {
-        output += line
 
-        currentSize += line.length
+      if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break
+    } else if (logTime < startTime) {
+      break
+    }
+  }
 
-        if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) stream.close()
+  return { currentSize, output: output.reverse(), logTime }
-      }
-    })
-  })
 }