Add pod list endpoint with pagination, sort...

README.md

@@ -59,7 +59,7 @@ Decentralized video streaming platform using P2P (BitTorrent) directly in the we
 Want to see in action?
 
    * [Demo server](http://peertube.cpy.re)
-   * [Video](https://vimeo.com/164881662 "Yes Vimeo, please don't judge me") to see how the "decentralization feature" looks like
+   * [Video](https://peertube.cpy.re/videos/watch/f78a97f8-a142-4ce1-a5bd-154bf9386504) to see how the "decentralization feature" looks like
    * Experimental demo servers that share videos (they are in the same network): [peertube2](http://peertube2.cpy.re), [peertube3](http://peertube3.cpy.re). Since I do experiments with them, sometimes they might not work correctly.
 
 ## Why

client/src/app/+admin/friends/friend-list/friend-list.component.html

@@ -2,12 +2,15 @@
   <div class="content-padding">
     <h3>Friends list</h3>
 
-    <p-dataTable [value]="friends">
-      <p-column field="id" header="ID"></p-column>
-      <p-column field="host" header="Host"></p-column>
+    <p-dataTable
+        [value]="friends" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage"
+        sortField="id" (onLazyLoad)="loadLazy($event)"
+    >
+      <p-column field="id" header="ID" [sortable]="true"></p-column>
+      <p-column field="host" header="Host" [sortable]="true"></p-column>
       <p-column field="email" header="Email"></p-column>
-      <p-column field="score" header="Score"></p-column>
-      <p-column field="createdAt" header="Created date"></p-column>
+      <p-column field="score" header="Score" [sortable]="true"></p-column>
+      <p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
       <p-column header="Delete" styleClass="action-cell">
         <ng-template pTemplate="body" let-pod="rowData">
           <span (click)="removeFriend(pod)" class="glyphicon glyphicon-remove glyphicon-black" title="Remove this pod"></span>

client/src/app/+admin/friends/friend-list/friend-list.component.ts

@@ -1,24 +1,32 @@
 import { Component, OnInit } from '@angular/core'
 
 import { NotificationsService } from 'angular2-notifications'
+import { SortMeta } from 'primeng/primeng'
 
 import { ConfirmService } from '../../../core'
-import { FriendService } from '../shared'
+import { RestTable, RestPagination } from '../../../shared'
 import { Pod } from '../../../../../../shared'
+import { FriendService } from '../shared'
 
 @Component({
   selector: 'my-friend-list',
   templateUrl: './friend-list.component.html',
   styleUrls: ['./friend-list.component.scss']
 })
-export class FriendListComponent implements OnInit {
+export class FriendListComponent extends RestTable implements OnInit {
   friends: Pod[] = []
+  totalRecords = 0
+  rowsPerPage = 10
+  sort: SortMeta = { field: 'id', order: 1 }
+  pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
 
   constructor (
     private notificationsService: NotificationsService,
     private confirmService: ConfirmService,
     private friendService: FriendService
-  ) {}
+  ) {
+    super()
+  }
 
   ngOnInit () {
     this.loadData()
@@ -65,11 +73,12 @@ export class FriendListComponent implements OnInit {
     )
   }
 
-  private loadData () {
-    this.friendService.getFriends()
+  protected loadData () {
+    this.friendService.getFriends(this.pagination, this.sort)
                       .subscribe(
                         resultList => {
                           this.friends = resultList.data
+                          this.totalRecords = resultList.total
                         },
 
                         err => this.notificationsService.error('Error', err.message)

client/src/app/+admin/friends/shared/friend.service.ts

@@ -1,9 +1,12 @@
 import { Injectable } from '@angular/core'
-import { HttpClient } from '@angular/common/http'
+import { HttpClient, HttpParams } from '@angular/common/http'
+import { Observable } from 'rxjs/Observable'
 import 'rxjs/add/operator/catch'
 import 'rxjs/add/operator/map'
 
-import { RestExtractor } from '../../../shared'
+import { SortMeta } from 'primeng/primeng'
+
+import { RestExtractor, RestPagination, RestService } from '../../../shared'
 import { Pod, ResultList } from '../../../../../../shared'
 
 @Injectable()
@@ -12,11 +15,15 @@ export class FriendService {
 
   constructor (
     private authHttp: HttpClient,
+    private restService: RestService,
     private restExtractor: RestExtractor
   ) {}
 
-  getFriends () {
-    return this.authHttp.get<ResultList<Pod>>(FriendService.BASE_FRIEND_URL)
+  getFriends (pagination: RestPagination, sort: SortMeta): Observable<ResultList<Pod>> {
+    let params = new HttpParams()
+    params = this.restService.addRestGetParams(params, pagination, sort)
+
+    return this.authHttp.get<ResultList<Pod>>(FriendService.BASE_FRIEND_URL, { params })
                         .map(res => this.restExtractor.convertResultListDateToHuman(res))
                         .catch(res => this.restExtractor.handleError(res))
   }

server/controllers/api/pods.ts

@@ -1,39 +1,33 @@
 import * as express from 'express'
 
 import { database as db } from '../../initializers/database'
-import { CONFIG } from '../../initializers'
+import { logger, getFormattedObjects } from '../../helpers'
 import {
-  logger,
-  getMyPublicCert,
-  getFormattedObjects
-} from '../../helpers'
-import {
-  sendOwnedVideosToPod,
   makeFriends,
   quitFriends,
   removeFriend
 } from '../../lib'
 import {
-  podsAddValidator,
   authenticate,
   ensureIsAdmin,
   makeFriendsValidator,
-  setBodyHostPort,
   setBodyHostsPort,
-  podRemoveValidator
+  podRemoveValidator,
+  paginationValidator,
+  setPagination,
+  setPodsSort,
+  podsSortValidator
 } from '../../middlewares'
-import {
-  PodInstance
-} from '../../models'
-import { Pod as FormattedPod } from '../../../shared'
+import { PodInstance } from '../../models'
 
 const podsRouter = express.Router()
 
-podsRouter.get('/', listPods)
-podsRouter.post('/',
-  setBodyHostPort, // We need to modify the host before running the validator!
-  podsAddValidator,
-  addPods
+podsRouter.get('/',
+  paginationValidator,
+  podsSortValidator,
+  setPodsSort,
+  setPagination,
+  listPods
 )
 podsRouter.post('/make-friends',
   authenticate,
@@ -62,26 +56,9 @@ export {
 
 // ---------------------------------------------------------------------------
 
-function addPods (req: express.Request, res: express.Response, next: express.NextFunction) {
-  const informations = req.body
-
-  const pod = db.Pod.build(informations)
-  pod.save()
-    .then(podCreated => {
-      return sendOwnedVideosToPod(podCreated.id)
-    })
-    .then(() => {
-      return getMyPublicCert()
-    })
-    .then(cert => {
-      return res.json({ cert: cert, email: CONFIG.ADMIN.EMAIL })
-    })
-    .catch(err => next(err))
-}
-
 function listPods (req: express.Request, res: express.Response, next: express.NextFunction) {
-  db.Pod.list()
-    .then(podsList => res.json(getFormattedObjects<FormattedPod, PodInstance>(podsList, podsList.length)))
+  db.Pod.listForApi(req.query.start, req.query.count, req.query.sort)
+    .then(resultList => res.json(getFormattedObjects(resultList.data, resultList.total)))
     .catch(err => next(err))
 }
 

server/controllers/api/remote/pods.ts

@@ -1,18 +1,34 @@
 import * as express from 'express'
 
 import { database as db } from '../../../initializers/database'
-import { checkSignature, signatureValidator } from '../../../middlewares'
-import { PodSignature } from '../../../../shared'
+import {
+  checkSignature,
+  signatureValidator,
+  setBodyHostPort,
+  remotePodsAddValidator
+} from '../../../middlewares'
+import { sendOwnedVideosToPod } from '../../../lib'
+import { getMyPublicCert, getFormattedObjects } from '../../../helpers'
+import { CONFIG } from '../../../initializers'
+import { PodInstance } from '../../../models'
+import { PodSignature, Pod as FormattedPod } from '../../../../shared'
 
 const remotePodsRouter = express.Router()
 
-// Post because this is a secured request
 remotePodsRouter.post('/remove',
   signatureValidator,
   checkSignature,
   removePods
 )
 
+remotePodsRouter.post('/list', remotePodsList)
+
+remotePodsRouter.post('/add',
+  setBodyHostPort, // We need to modify the host before running the validator!
+  remotePodsAddValidator,
+  addPods
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -21,6 +37,29 @@ export {
 
 // ---------------------------------------------------------------------------
 
+function addPods (req: express.Request, res: express.Response, next: express.NextFunction) {
+  const information = req.body
+
+  const pod = db.Pod.build(information)
+  pod.save()
+     .then(podCreated => {
+       return sendOwnedVideosToPod(podCreated.id)
+     })
+     .then(() => {
+       return getMyPublicCert()
+     })
+     .then(cert => {
+       return res.json({ cert: cert, email: CONFIG.ADMIN.EMAIL })
+     })
+     .catch(err => next(err))
+}
+
+function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) {
+  db.Pod.list()
+    .then(podsList => res.json(getFormattedObjects<FormattedPod, PodInstance>(podsList, podsList.length)))
+    .catch(err => next(err))
+}
+
 function removePods (req: express.Request, res: express.Response, next: express.NextFunction) {
   const signature: PodSignature = req.body.signature
   const host = signature.host

server/initializers/constants.ts

@@ -32,6 +32,7 @@ const SEARCHABLE_COLUMNS = {
 
 // Sortable columns per schema
 const SORTABLE_COLUMNS = {
+  PODS: [ 'id', 'host', 'score', 'createdAt' ],
   USERS: [ 'id', 'username', 'createdAt' ],
   VIDEO_ABUSES: [ 'id', 'createdAt' ],
   VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ],

server/lib/friends.ts

@@ -334,9 +334,9 @@ function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: num
 
 function getForeignPodsList (host: string) {
   return new Promise< ResultList<FormattedPod> >((res, rej) => {
-    const path = '/api/' + API_VERSION + '/pods'
+    const path = '/api/' + API_VERSION + '/remote/pods/list'
 
-    request.get(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
+    request.post(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
       if (err) return rej(err)
 
       try {
@@ -357,7 +357,7 @@ function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
 
   return Promise.map(podsList, pod => {
     const params = {
-      url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/',
+      url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add',
       method: 'POST' as 'POST',
       json: {
         host: CONFIG.WEBSERVER.HOST,

server/middlewares/sort.ts

@@ -4,6 +4,12 @@ import * as express from 'express'
 import { SortType } from '../helpers'
 import { database } from '../initializers'
 
+function setPodsSort (req: express.Request, res: express.Response, next: express.NextFunction) {
+  if (!req.query.sort) req.query.sort = '-createdAt'
+
+  return next()
+}
+
 function setUsersSort (req: express.Request, res: express.Response, next: express.NextFunction) {
   if (!req.query.sort) req.query.sort = '-createdAt'
 
@@ -46,6 +52,7 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex
 // ---------------------------------------------------------------------------
 
 export {
+  setPodsSort,
   setUsersSort,
   setVideoAbusesSort,
   setVideosSort,

server/middlewares/validators/pods.ts

@@ -3,7 +3,7 @@ import * as express from 'express'
 
 import { database as db } from '../../initializers/database'
 import { checkErrors } from './utils'
-import { logger, isEachUniqueHostValid, isHostValid } from '../../helpers'
+import { logger, isEachUniqueHostValid } from '../../helpers'
 import { CONFIG } from '../../initializers'
 import { hasFriends } from '../../lib'
 import { isTestInstance } from '../../helpers'
@@ -41,32 +41,6 @@ const makeFriendsValidator = [
   }
 ]
 
-const podsAddValidator = [
-  body('host').custom(isHostValid).withMessage('Should have a host'),
-  body('email').isEmail().withMessage('Should have an email'),
-  body('publicKey').not().isEmpty().withMessage('Should have a public key'),
-
-  (req: express.Request, res: express.Response, next: express.NextFunction) => {
-    logger.debug('Checking podsAdd parameters', { parameters: req.body })
-
-    checkErrors(req, res, () => {
-      db.Pod.loadByHost(req.body.host)
-        .then(pod => {
-          // Pod with this host already exists
-          if (pod) {
-            return res.sendStatus(409)
-          }
-
-          return next()
-        })
-        .catch(err => {
-          logger.error('Cannot load pod by host.', err)
-          res.sendStatus(500)
-        })
-    })
-  }
-]
-
 const podRemoveValidator = [
   param('id').isNumeric().not().isEmpty().withMessage('Should have a valid id'),
 
@@ -96,6 +70,5 @@ const podRemoveValidator = [
 
 export {
   makeFriendsValidator,
-  podsAddValidator,
   podRemoveValidator
 }

server/middlewares/validators/remote/index.ts

@@ -1,2 +1,3 @@
+export * from './pods'
 export * from './signature'
 export * from './videos'

server/middlewares/validators/remote/pods.ts

@@ -0,0 +1,38 @@
+import { body } from 'express-validator/check'
+import * as express from 'express'
+
+import { database as db } from '../../../initializers'
+import { isHostValid, logger } from '../../../helpers'
+import { checkErrors } from '../utils'
+
+const remotePodsAddValidator = [
+  body('host').custom(isHostValid).withMessage('Should have a host'),
+  body('email').isEmail().withMessage('Should have an email'),
+  body('publicKey').not().isEmpty().withMessage('Should have a public key'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking podsAdd parameters', { parameters: req.body })
+
+    checkErrors(req, res, () => {
+      db.Pod.loadByHost(req.body.host)
+        .then(pod => {
+          // Pod with this host already exists
+          if (pod) {
+            return res.sendStatus(409)
+          }
+
+          return next()
+        })
+        .catch(err => {
+          logger.error('Cannot load pod by host.', err)
+          res.sendStatus(500)
+        })
+    })
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  remotePodsAddValidator
+}

server/middlewares/validators/sort.ts

@@ -6,11 +6,13 @@ import { logger } from '../../helpers'
 import { SORTABLE_COLUMNS } from '../../initializers'
 
 // Initialize constants here for better performances
+const SORTABLE_PODS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PODS)
 const SORTABLE_USERS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.USERS)
 const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES)
 const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS)
 const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS)
 
+const podsSortValidator = checkSort(SORTABLE_PODS_COLUMNS)
 const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
 const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS)
 const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS)
@@ -19,6 +21,7 @@ const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS)
 // ---------------------------------------------------------------------------
 
 export {
+  podsSortValidator,
   usersSortValidator,
   videoAbusesSortValidator,
   videosSortValidator,

server/middlewares/validators/utils.ts

@@ -3,12 +3,12 @@ import * as express from 'express'
 
 import { logger } from '../../helpers'
 
-function checkErrors (req: express.Request, res: express.Response, next: express.NextFunction, statusCode = 400) {
+function checkErrors (req: express.Request, res: express.Response, next: express.NextFunction) {
   const errors = validationResult(req)
 
   if (!errors.isEmpty()) {
     logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors.mapped() })
-    return res.status(statusCode).json({ errors: errors.mapped() })
+    return res.status(400).json({ errors: errors.mapped() })
   }
 
   return next()

server/models/pod/pod-interface.ts

@@ -3,6 +3,7 @@ import * as Promise from 'bluebird'
 
 // Don't use barrel, import just what we need
 import { Pod as FormattedPod } from '../../../shared/models/pods/pod.model'
+import { ResultList } from '../../../shared/models/result-list.model'
 
 export namespace PodMethods {
   export type ToFormattedJSON = (this: PodInstance) => FormattedPod
@@ -13,6 +14,8 @@ export namespace PodMethods {
 
   export type List = () => Promise<PodInstance[]>
 
+  export type ListForApi = (start: number, count: number, sort: string) => Promise< ResultList<PodInstance> >
+
   export type ListAllIds = (transaction: Sequelize.Transaction) => Promise<number[]>
 
   export type ListRandomPodIdsWithRequest = (limit: number, tableWithPods: string, tableWithPodsJoins: string) => Promise<number[]>
@@ -32,6 +35,7 @@ export interface PodClass {
   countAll: PodMethods.CountAll
   incrementScores: PodMethods.IncrementScores
   list: PodMethods.List
+  listForApi: PodMethods.ListForApi
   listAllIds: PodMethods.ListAllIds
   listRandomPodIdsWithRequest: PodMethods.ListRandomPodIdsWithRequest
   listBadPods: PodMethods.ListBadPods

server/models/pod/pod.ts

@@ -4,7 +4,7 @@ import * as Sequelize from 'sequelize'
 import { FRIEND_SCORE, PODS_SCORE } from '../../initializers'
 import { logger, isHostValid } from '../../helpers'
 
-import { addMethodsToModel } from '../utils'
+import { addMethodsToModel, getSort } from '../utils'
 import {
   PodInstance,
   PodAttributes,
@@ -17,6 +17,7 @@ let toFormattedJSON: PodMethods.ToFormattedJSON
 let countAll: PodMethods.CountAll
 let incrementScores: PodMethods.IncrementScores
 let list: PodMethods.List
+let listForApi: PodMethods.ListForApi
 let listAllIds: PodMethods.ListAllIds
 let listRandomPodIdsWithRequest: PodMethods.ListRandomPodIdsWithRequest
 let listBadPods: PodMethods.ListBadPods
@@ -78,6 +79,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
     countAll,
     incrementScores,
     list,
+    listForApi,
     listAllIds,
     listRandomPodIdsWithRequest,
     listBadPods,
@@ -142,6 +144,21 @@ list = function () {
   return Pod.findAll()
 }
 
+listForApi = function (start: number, count: number, sort: string) {
+  const query = {
+    offset: start,
+    limit: count,
+    order: [ getSort(sort) ]
+  }
+
+  return Pod.findAndCountAll(query).then(({ rows, count }) => {
+    return {
+      data: rows,
+      total: count
+    }
+  })
+}
+
 listAllIds = function (transaction: Sequelize.Transaction) {
   const query = {
     attributes: [ 'id' ],

server/tests/api/check-params/pods.ts

@@ -15,7 +15,6 @@ import {
 } from '../../utils'
 
 describe('Test pods API validators', function () {
-  const path = '/api/v1/pods/'
   let server: ServerInfo
 
   // ---------------------------------------------------------------
@@ -30,6 +29,7 @@ describe('Test pods API validators', function () {
   })
 
   describe('When managing friends', function () {
+    const path = '/api/v1/pods/'
     let userAccessToken = null
 
     before(async function () {
@@ -110,6 +110,32 @@ describe('Test pods API validators', function () {
       })
     })
 
+    describe('When listing friends', function () {
+      it('Should fail with a bad start pagination', async function () {
+        await request(server.url)
+          .get(path)
+          .query({ start: 'hello' })
+          .set('Accept', 'application/json')
+          .expect(400)
+      })
+
+      it('Should fail with a bad count pagination', async function () {
+        await request(server.url)
+          .get(path)
+          .query({ count: 'hello' })
+          .set('Accept', 'application/json')
+          .expect(400)
+      })
+
+      it('Should fail with an incorrect sort', async function () {
+        await request(server.url)
+          .get(path)
+          .query({ sort: 'hello' })
+          .set('Accept', 'application/json')
+          .expect(400)
+      })
+    })
+
     describe('When quitting friends', function () {
       it('Should fail with an invalid token', async function () {
         await request(server.url)
@@ -175,7 +201,9 @@ describe('Test pods API validators', function () {
     })
   })
 
-  describe('When adding a pod', function () {
+  describe('When adding a pod from remote', function () {
+    const path = '/api/v1/remote/pods/add'
+
     it('Should fail with nothing', async function () {
       const fields = {}
       await makePostBodyRequest({ url: server.url, path, fields })

server/tests/api/friends-basic.ts

@@ -15,7 +15,8 @@ import {
   makeFriends,
   getFriendsList,
   dateIsValid,
-  quitOneFriend
+  quitOneFriend,
+  getPodsListPaginationAndSort
 } from '../utils'
 
 describe('Test basic friends', function () {
@@ -120,6 +121,22 @@ describe('Test basic friends', function () {
     await makeFriends(server.url, server.accessToken, 409)
   })
 
+  it('Should list friends correctly', async function () {
+    const start = 1
+    const count = 1
+    const sort = '-host'
+
+    const res = await getPodsListPaginationAndSort(servers[0].url, start, count, sort)
+    expect(res.body.total).to.equal(2)
+    expect(res.body.data).to.have.lengthOf(1)
+
+    const pod = res.body.data[0]
+    expect(pod.host).to.equal('localhost:9002')
+    expect(pod.email).to.equal('admin2@example.com')
+    expect(pod.score).to.equal(20)
+    expect(dateIsValid(pod.createdAt)).to.be.true
+  })
+
   it('Should quit friends of pod 2', async function () {
     this.timeout(10000)
 

server/tests/utils/pods.ts

@@ -12,6 +12,19 @@ function getFriendsList (url: string) {
           .expect('Content-Type', /json/)
 }
 
+function getPodsListPaginationAndSort (url: string, start: number, count: number, sort: string) {
+  const path = '/api/v1/pods/'
+
+  return request(url)
+    .get(path)
+    .query({ start })
+    .query({ count })
+    .query({ sort })
+    .set('Accept', 'application/json')
+    .expect(200)
+    .expect('Content-Type', /json/)
+}
+
 async function makeFriends (url: string, accessToken: string, expectedStatus = 204) {
   // Which pod makes friends with which pod
   const friendsMatrix = {
@@ -85,5 +98,6 @@ export {
   getFriendsList,
   makeFriends,
   quitFriends,
-  quitOneFriend
+  quitOneFriend,
+  getPodsListPaginationAndSort
 }