Improve update host script and add warning if AP urls are invalid

scripts/update-host.ts

@@ -1,35 +1,126 @@
-import { getServerActor } from '../server/helpers/utils'
+import { CONFIG, initDatabaseModels } from '../server/initializers'
-import { initDatabaseModels } from '../server/initializers'
 import { ActorFollowModel } from '../server/models/activitypub/actor-follow'
 import { VideoModel } from '../server/models/video/video'
+import { ActorModel } from '../server/models/activitypub/actor'
+import {
+  getAccountActivityPubUrl,
+  getAnnounceActivityPubUrl,
+  getVideoActivityPubUrl, getVideoChannelActivityPubUrl,
+  getVideoCommentActivityPubUrl
+} from '../server/lib/activitypub'
+import { VideoShareModel } from '../server/models/video/video-share'
+import { VideoCommentModel } from '../server/models/video/video-comment'
+import { getServerActor } from '../server/helpers/utils'
+import { AccountModel } from '../server/models/account/account'
+import { VideoChannelModel } from '../server/models/video/video-channel'
 
-initDatabaseModels(true)
+run()
-  .then(() => {
+  .then(() => process.exit(0))
-    return getServerActor()
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
   })
-  .then(serverAccount => {
+
-    return ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined)
+async function run () {
-  })
+  await initDatabaseModels(true)
-  .then(res => {
+
-    return res.total > 0
+  const serverAccount = await getServerActor()
-  })
+
-  .then(hasFollowing => {
+  {
+    const res = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined)
+    const hasFollowing = res.total > 0
+
     if (hasFollowing === true) {
-      console.log('Cannot update host because you follow other servers!')
+      throw new Error('Cannot update host because you follow other servers!')
-      process.exit(-1)
     }
+  }
 
-    console.log('Updating torrent files.')
+  console.log('Updating actors.')
-    return VideoModel.list()
+
-  })
+  const actors: ActorModel[] = await ActorModel.unscoped().findAll({
-  .then(async videos => {
+    include: [
-    for (const video of videos) {
+      {
-      for (const file of video.VideoFiles) {
+        model: VideoChannelModel.unscoped(),
-        await video.createTorrentAndSetInfoHash(file)
+        required: false
-        console.log('Updated video ' + video.uuid)
+      },
+      {
+        model: AccountModel.unscoped(),
+        required: false
       }
+    ]
+  })
+  for (const actor of actors) {
+    if (actor.isOwned() === false) continue
+
+    console.log('Updating actor ' + actor.url)
+
+    const newUrl = actor.Account
+      ? getAccountActivityPubUrl(actor.preferredUsername)
+      : getVideoChannelActivityPubUrl(actor.preferredUsername)
+
+    actor.url = newUrl
+    actor.inboxUrl = newUrl + '/inbox'
+    actor.outboxUrl = newUrl + '/outbox'
+    actor.sharedInboxUrl = CONFIG.WEBSERVER.URL + '/inbox'
+    actor.followersUrl = newUrl + '/followers'
+    actor.followingUrl = newUrl + '/following'
+
+    await actor.save()
+  }
+
+  console.log('Updating video shares.')
+
+  const videoShares: VideoShareModel[] = await VideoShareModel.findAll({
+    include: [ VideoModel.unscoped(), ActorModel.unscoped() ]
+  })
+  for (const videoShare of videoShares) {
+    if (videoShare.Video.isOwned() === false) continue
+
+    console.log('Updating video share ' + videoShare.url)
+
+    videoShare.url = getAnnounceActivityPubUrl(videoShare.Video.url, videoShare.Actor)
+    await videoShare.save()
+  }
+
+  console.log('Updating video comments.')
+  const videoComments: VideoCommentModel[] = await VideoCommentModel.findAll({
+    include: [
+      {
+        model: VideoModel.unscoped()
+      },
+      {
+        model: AccountModel.unscoped(),
+        include: [
+          {
+            model: ActorModel.unscoped()
+          }
+        ]
+      }
+    ]
+  })
+  for (const comment of videoComments) {
+    if (comment.isOwned() === false) continue
+
+    console.log('Updating comment ' + comment.url)
+
+    comment.url = getVideoCommentActivityPubUrl(comment.Video, comment)
+    await comment.save()
+  }
+
+  console.log('Updating video and torrent files.')
+
+  const videos = await VideoModel.list()
+  for (const video of videos) {
+    if (video.isOwned() === false) continue
+
+    console.log('Updated video ' + video.uuid)
+
+    video.url = getVideoActivityPubUrl(video)
+    await video.save()
+
+    for (const file of video.VideoFiles) {
+      console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
+      await video.createTorrentAndSetInfoHash(file)
     }
-  })
+  }
-  .then(() => {
+}
-    process.exit(0)
-  })

server.ts

@@ -1,6 +1,4 @@
 // FIXME: https://github.com/nodejs/node/pull/16853
-import { ScheduleVideoUpdateModel } from './server/models/video/schedule-video-update'
-
 require('tls').DEFAULT_ECDH_CURVE = 'auto'
 
 import { isTestInstance } from './server/helpers/core-utils'
@@ -26,7 +24,7 @@ process.title = 'peertube'
 const app = express()
 
 // ----------- Core checker -----------
-import { checkMissedConfig, checkFFmpeg, checkConfig } from './server/initializers/checker'
+import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } from './server/initializers/checker'
 
 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
 import { logger } from './server/helpers/logger'
@@ -191,6 +189,13 @@ async function startApplication () {
 
   await installApplication()
 
+  // Check activity pub urls are valid
+  checkActivityPubUrls()
+    .catch(err => {
+      logger.error('Error in ActivityPub URLs checker.', { err })
+      process.exit(-1)
+    })
+
   // Email initialization
   Emailer.Instance.init()
   await Emailer.Instance.checkConnectionOrDie()

server/initializers/checker.ts

@@ -3,6 +3,28 @@ import { promisify0 } from '../helpers/core-utils'
 import { UserModel } from '../models/account/user'
 import { ApplicationModel } from '../models/application/application'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
+import { parse } from 'url'
+import { CONFIG } from './constants'
+import { logger } from '../helpers/logger'
+import { getServerActor } from '../helpers/utils'
+
+async function checkActivityPubUrls () {
+  const actor = await getServerActor()
+
+  const parsed = parse(actor.url)
+  if (CONFIG.WEBSERVER.HOST !== parsed.host) {
+    const NODE_ENV = config.util.getEnv('NODE_ENV')
+    const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
+
+    logger.warn(
+      'It seems PeerTube was started (and created some data) with another domain name. ' +
+      'This means you will not be able to federate! ' +
+      'Please use %s %s npm run update-host to fix this.',
+      NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
+      NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
+    )
+  }
+}
 
 // Some checks on configuration files
 // Return an error message, or null if everything is okay
@@ -95,5 +117,6 @@ export {
   checkMissedConfig,
   clientsExist,
   usersExist,
-  applicationExist
+  applicationExist,
+  checkActivityPubUrls
 }

server/tests/cli/update-host.ts

@@ -3,20 +3,26 @@
 import 'mocha'
 import * as chai from 'chai'
 import { VideoDetails } from '../../../shared/models/videos'
+import { waitJobs } from '../utils/server/jobs'
+import { addVideoCommentThread } from '../utils/videos/video-comments'
 import {
+  addVideoChannel,
+  createUser,
   execCLI,
   flushTests,
   getEnvCli,
   getVideo,
+  getVideoChannelsList,
   getVideosList,
   killallServers,
+  makeActivityPubGetRequest,
   parseTorrentVideo,
   runServer,
   ServerInfo,
   setAccessTokensToServers,
   uploadVideo
 } from '../utils'
-import { waitJobs } from '../utils/server/jobs'
+import { getAccountsList } from '../utils/users/accounts'
 
 const expect = chai.expect
 
@@ -39,13 +45,28 @@ describe('Test update host scripts', function () {
 
     // Upload two videos for our needs
     const videoAttributes = {}
+    const resVideo1 = await uploadVideo(server.url, server.accessToken, videoAttributes)
+    const video1UUID = resVideo1.body.video.uuid
     await uploadVideo(server.url, server.accessToken, videoAttributes)
-    await uploadVideo(server.url, server.accessToken, videoAttributes)
+
+    // Create a user
+    await createUser(server.url, server.accessToken, 'toto', 'coucou')
+
+    // Create channel
+    const videoChannel = {
+      displayName: 'second video channel',
+      description: 'super video channel description'
+    }
+    await addVideoChannel(server.url, server.accessToken, videoChannel)
+
+    // Create comments
+    const text = 'my super first comment'
+    await addVideoCommentThread(server.url, server.accessToken, video1UUID, text)
 
     await waitJobs(server)
   })
 
-  it('Should update torrent hosts', async function () {
+  it('Should run update host', async function () {
     this.timeout(30000)
 
     killallServers([ server ])
@@ -54,6 +75,44 @@ describe('Test update host scripts', function () {
 
     const env = getEnvCli(server)
     await execCLI(`${env} npm run update-host`)
+  })
+
+  it('Should have updated videos url', async function () {
+    const res = await getVideosList(server.url)
+    expect(res.body.total).to.equal(2)
+
+    for (const video of res.body.data) {
+      const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
+
+      expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
+    }
+  })
+
+  it('Should have updated video channels url', async function () {
+    const res = await getVideoChannelsList(server.url, 0, 5, '-name')
+    expect(res.body.total).to.equal(3)
+
+    for (const channel of res.body.data) {
+      const { body } = await makeActivityPubGetRequest(server.url, '/video-channels/' + channel.uuid)
+
+      expect(body.id).to.equal('http://localhost:9002/video-channels/' + channel.uuid)
+    }
+  })
+
+  it('Should have update accounts url', async function () {
+    const res = await getAccountsList(server.url)
+    expect(res.body.total).to.equal(3)
+
+    for (const account of res.body.data) {
+      const usernameWithDomain = account.name
+      const { body } = await makeActivityPubGetRequest(server.url, '/accounts/' + usernameWithDomain)
+
+      expect(body.id).to.equal('http://localhost:9002/accounts/' + usernameWithDomain)
+    }
+  })
+
+  it('Should update torrent hosts', async function () {
+    this.timeout(30000)
 
     const res = await getVideosList(server.url)
     const videos = res.body.data

support/doc/tools.md

@@ -113,4 +113,13 @@ To delete them (a confirmation will be demanded first):
 
 ```
 $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage
+```
+
+### update-host.js
+
+If you started PeerTube with a domain, and then changed it you will have invalid torrent files and invalid URLs in your database.
+To fix this, you have to run:
+
+```
+$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-host
 ```

support/doc/translation.md

@@ -18,7 +18,8 @@ If you don't see your locale in the platform, please [create an issue](https://g
 
 There are 4 files:
  * **angular**: contains client strings
- * **player**: contains player strings
+ * **player**: contains player strings. 
- * **server**: contains server strings (language, licence...)
+ Most of the strings come from VideoJS, so you can help yourself by using [video.js JSON files](https://github.com/videojs/video.js/tree/master/lang)
+ * **server**: contains server strings (privacies, licences...)
  * **iso639**: contains iso639 (languages) strings used by PeerTube to describe the audio language of a particular video.
  It's the reason why these strings should be translated too. There are many strings so do not hesitate to translate only main audio languages.