Implement user import/export in server

config/default.yaml

@@ -1,4 +1,4 @@
-# /!\ YOU SHOULD NOT UPDATE THIS FILE, USE production.yaml instead /!\ #
+# /!\ DO NOT UPDATE THIS FILE, USE production.yaml instead /!\ #
 
 listen:
   hostname: '127.0.0.1'
@@ -222,12 +222,16 @@ object_storage:
     # Useful when you want to use a CDN/external proxy
     base_url: '' # Example: 'https://mirror.example.com'
 
-  # Same settings but for web videos
   web_videos:
     bucket_name: 'web-videos'
     prefix: ''
     base_url: ''
 
+  user_exports:
+    bucket_name: 'user-exports'
+    prefix: ''
+    base_url: ''
+
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'
 
@@ -482,11 +486,14 @@ user:
     videos:
       # Enable or disable video history by default for new users.
       enabled: true
-  # Default value of maximum video bytes the user can upload (does not take into account transcoded files)
+
+  # Default value of maximum video bytes the user can upload
+  # Does not take into account transcoded files or account export archives (that can include user uploaded files)
   # Byte format is supported ("1GB" etc)
   # -1 == unlimited
   video_quota: -1
   video_quota_daily: -1
+
   default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username
 
 video_channels:
@@ -707,6 +714,24 @@ import:
     # Max number of videos to import when the user asks for full sync
     full_sync_videos_limit: 1000
 
+  users:
+    # Video quota is checked on import so the user doesn't upload a too big archive file
+    # Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import
+    enabled: true
+
+export:
+  users:
+    # Allow users to export their PeerTube data in a .zip for backup or re-import
+    # Only one export at a time is allowed per user
+    enabled: true
+
+    # Max size of the current user quota to accept or not the export
+    # Goal of this setting is to not store too big archive file on your server disk
+    max_user_video_quota: 10GB
+
+    # How long PeerTube should keep the user export
+    export_expiration: '2 days'
+
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing
   videos:
@@ -867,6 +892,7 @@ client:
       # By default PeerTube client displays author username
       prefer_author_display_name: false
       display_author_avatar: false
+
     resumable_upload:
       # Max size of upload chunks, e.g. '90MB'
       # If null, it will be calculated based on network speed

config/production.yaml.example

@@ -220,12 +220,16 @@ object_storage:
     # Useful when you want to use a CDN/external proxy
     base_url: '' # Example: 'https://mirror.example.com'
 
-  # Same settings but for web videos
   web_videos:
     bucket_name: 'web-videos'
     prefix: ''
     base_url: ''
 
+  user_exports:
+    bucket_name: 'user-exports'
+    prefix: ''
+    base_url: ''
+
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'
 
@@ -492,11 +496,14 @@ user:
     videos:
       # Enable or disable video history by default for new users.
       enabled: true
-  # Default value of maximum video bytes the user can upload (does not take into account transcoded files)
+
+  # Default value of maximum video bytes the user can upload
+  # Does not take into account transcoded files or account export archives (that can include user uploaded files)
   # Byte format is supported ("1GB" etc)
   # -1 == unlimited
   video_quota: -1
   video_quota_daily: -1
+
   default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username
 
 video_channels:
@@ -717,6 +724,24 @@ import:
     # Max number of videos to import when the user asks for full sync
     full_sync_videos_limit: 1000
 
+  users:
+    # Video quota is checked on import so the user doesn't upload a too big archive file
+    # Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import
+    enabled: true
+
+export:
+  users:
+    # Allow users to export their PeerTube data in a .zip for backup or re-import
+    # Only one export at a time is allowed per user
+    enabled: true
+
+    # Max size of the current user quota to accept or not the export
+    # Goal of this setting is to not store too big archive file on your server disk
+    max_user_video_quota: 10GB
+
+    # How long PeerTube should keep the user export
+    export_expiration: '2 days'
+
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing
   videos:
@@ -877,6 +902,7 @@ client:
       # By default PeerTube client displays author username
       prefer_author_display_name: false
       display_author_avatar: false
+
     resumable_upload:
       # Max size of upload chunks, e.g. '90MB'
       # If null, it will be calculated based on network speed

package.json

@@ -109,6 +109,7 @@
     "@peertube/http-signature": "^1.7.0",
     "@smithy/node-http-handler": "^2.1.7",
     "@uploadx/core": "^6.0.0",
+    "archiver": "^6.0.1",
     "async-mutex": "^0.4.0",
     "bcrypt": "5.1.1",
     "bencode": "^4.0.0",
@@ -142,6 +143,7 @@
     "jimp": "^0.22.4",
     "js-yaml": "^4.0.0",
     "jsonld": "~8.3.1",
+    "jsonwebtoken": "^9.0.2",
     "lodash-es": "^4.17.21",
     "lru-cache": "^10.0.1",
     "magnet-uri": "^7.0.5",
@@ -178,11 +180,13 @@
     "webfinger.js": "^2.6.6",
     "webtorrent": "^2.1.27",
     "winston": "3.11.0",
-    "ws": "^8.0.0"
+    "ws": "^8.0.0",
+    "yauzl": "^2.10.0"
   },
   "devDependencies": {
     "@peertube/maildev": "^1.2.0",
     "@peertube/resolve-tspaths": "^0.8.14",
+    "@types/archiver": "^6.0.2",
     "@types/bcrypt": "^5.0.0",
     "@types/bencode": "^2.0.0",
     "@types/bluebird": "^3.5.33",
@@ -197,6 +201,7 @@
     "@types/fluent-ffmpeg": "^2.1.16",
     "@types/fs-extra": "^11.0.1",
     "@types/jsonld": "^1.5.9",
+    "@types/jsonwebtoken": "^9.0.5",
     "@types/lodash-es": "^4.17.8",
     "@types/magnet-uri": "^5.1.1",
     "@types/maildev": "^0.0.4",
@@ -212,6 +217,7 @@
     "@types/validator": "^13.9.0",
     "@types/webtorrent": "^0.109.0",
     "@types/ws": "^8.2.0",
+    "@types/yauzl": "^2.10.3",
     "@typescript-eslint/eslint-plugin": "^6.7.5",
     "autocannon": "^7.0.4",
     "chai": "^4.1.1",
@@ -228,6 +234,7 @@
     "eslint-plugin-promise": "^6.0.0",
     "fast-xml-parser": "^4.0.0-beta.8",
     "jpeg-js": "^0.4.4",
+    "jszip": "^3.10.1",
     "mocha": "^10.0.0",
     "pixelmatch": "^5.3.0",
     "pngjs": "^7.0.0",

packages/core-utils/src/common/url.ts

@@ -19,6 +19,18 @@ function removeQueryParams (url: string) {
   return objUrl.toString()
 }
 
+function queryParamsToObject (entries: any) {
+  const result: { [ id: string ]: string | number | boolean } = {}
+
+  for (const [ key, value ] of entries) {
+    result[key] = value
+  }
+
+  return result
+}
+
+// ---------------------------------------------------------------------------
+
 function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) {
   return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist)
 }
@@ -123,6 +135,7 @@ function decoratePlaylistLink (options: {
 export {
   addQueryParams,
   removeQueryParams,
+  queryParamsToObject,
 
   buildPlaylistLink,
   buildVideoLink,

packages/models/src/activitypub/activitypub-actor.ts

@@ -18,7 +18,7 @@ export interface ActivityPubActor {
     sharedInbox: string
   }
   summary: string
-  attributedTo: ActivityPubAttributedTo[]
+  attributedTo?: ActivityPubAttributedTo[]
 
   support?: string
   publicKey: {
@@ -31,4 +31,8 @@ export interface ActivityPubActor {
   icon?: ActivityIconObject | ActivityIconObject[]
 
   published?: string
+
+  // For export
+  likes?: string
+  dislikes?: string
 }

packages/models/src/activitypub/activitypub-collection.ts

@@ -1,7 +1,7 @@
 import { Activity } from './activity.js'
 
 export interface ActivityPubCollection {
-  '@context': string[]
+  '@context': any[]
   type: 'Collection' | 'CollectionPage'
   totalItems: number
   partOf?: string

packages/models/src/activitypub/activitypub-ordered-collection.ts

@@ -1,5 +1,7 @@
 export interface ActivityPubOrderedCollection<T> {
-  '@context': string[]
+  id: string
+
+  '@context': any[]
   type: 'OrderedCollection' | 'OrderedCollectionPage'
   totalItems: number
   orderedItems: T[]

packages/models/src/activitypub/objects/video-object.ts

@@ -59,6 +59,16 @@ export interface VideoObject {
 
   to?: string[]
   cc?: string[]
+
+  // For export
+  attachment?: {
+    type: 'Video'
+    url: string
+    mediaType: string
+    height: number
+    size: number
+    fps: number
+  }[]
 }
 
 export interface ActivityPubStoryboard {

packages/models/src/common/file-storage.enum.ts

@@ -0,0 +1,6 @@
+export const FileStorage = {
+  FILE_SYSTEM: 0,
+  OBJECT_STORAGE: 1
+} as const
+
+export type FileStorageType = typeof FileStorage[keyof typeof FileStorage]

packages/models/src/common/index.ts

@@ -1 +1,2 @@
+export * from './file-storage.enum.js'
 export * from './result-list.model.js'

packages/models/src/import-export/index.ts

@@ -0,0 +1,9 @@
+export * from './peertube-export-format/index.js'
+export * from './user-export-request-result.model.js'
+export * from './user-export-request.model.js'
+export * from './user-export-state.enum.js'
+export * from './user-export.model.js'
+export * from './user-import.model.js'
+export * from './user-import-state.enum.js'
+export * from './user-import-result.model.js'
+export * from './user-import-upload-result.model.js'

packages/models/src/import-export/peertube-export-format/account-export.model.ts

@@ -0,0 +1,18 @@
+import { UserActorImageJSON } from './actor-export.model.js'
+
+export interface AccountExportJSON {
+  url: string
+
+  name: string
+  displayName: string
+  description: string
+
+  updatedAt: string
+  createdAt: string
+
+  avatars: UserActorImageJSON[]
+
+  archiveFiles: {
+    avatar: string | null
+  }
+}

packages/models/src/import-export/peertube-export-format/actor-export.model.ts

@@ -0,0 +1,6 @@
+export interface UserActorImageJSON {
+  width: number
+  url: string
+  createdAt: string
+  updatedAt: string
+}

packages/models/src/import-export/peertube-export-format/blocklist-export.model.ts

@@ -0,0 +1,9 @@
+export interface BlocklistExportJSON {
+  instances: {
+    host: string
+  }[]
+
+  actors: {
+    handle: string
+  }[]
+}

packages/models/src/import-export/peertube-export-format/channel-export.model.ts

@@ -0,0 +1,23 @@
+import { UserActorImageJSON } from './actor-export.model.js'
+
+export interface ChannelExportJSON {
+  channels: {
+    url: string
+
+    name: string
+    displayName: string
+    description: string
+    support: string
+
+    updatedAt: string
+    createdAt: string
+
+    avatars: UserActorImageJSON[]
+    banners: UserActorImageJSON[]
+
+    archiveFiles: {
+      avatar: string | null
+      banner: string | null
+    }
+  }[]
+}

packages/models/src/import-export/peertube-export-format/comments-export.model.ts

@@ -0,0 +1,12 @@
+export interface CommentsExportJSON {
+  comments: {
+    url: string
+    text: string
+    createdAt: string
+    videoUrl: string
+
+    inReplyToCommentUrl?: string
+
+    archiveFiles?: never
+  }[]
+}

packages/models/src/import-export/peertube-export-format/dislikes-export.model.ts

@@ -0,0 +1,8 @@
+export interface DislikesExportJSON {
+  dislikes: {
+    videoUrl: string
+    createdAt: string
+
+    archiveFiles?: never
+  }[]
+}

packages/models/src/import-export/peertube-export-format/followers-export.model.ts

@@ -0,0 +1,9 @@
+export interface FollowersExportJSON {
+  followers: {
+    handle: string
+    createdAt: string
+    targetHandle: string
+
+    archiveFiles?: never
+  }[]
+}

packages/models/src/import-export/peertube-export-format/following-export.model.ts

@@ -0,0 +1,9 @@
+export interface FollowingExportJSON {
+  following: {
+    handle: string
+    targetHandle: string
+    createdAt: string
+
+    archiveFiles?: never
+  }[]
+}

packages/models/src/import-export/peertube-export-format/index.ts

@@ -0,0 +1,12 @@
+export * from './account-export.model.js'
+export * from './actor-export.model.js'
+export * from './blocklist-export.model.js'
+export * from './channel-export.model.js'
+export * from './comments-export.model.js'
+export * from './dislikes-export.model.js'
+export * from './followers-export.model.js'
+export * from './following-export.model.js'
+export * from './likes-export.model.js'
+export * from './user-settings-export.model.js'
+export * from './video-export.model.js'
+export * from './video-playlists-export.model.js'

packages/models/src/import-export/peertube-export-format/likes-export.model.ts

@@ -0,0 +1,8 @@
+export interface LikesExportJSON {
+  likes: {
+    videoUrl: string
+    createdAt: string
+
+    archiveFiles?: never
+  }[]
+}

packages/models/src/import-export/peertube-export-format/user-settings-export.model.ts

@@ -0,0 +1,26 @@
+import { UserNotificationSetting } from '../../users/user-notification-setting.model.js'
+import { NSFWPolicyType } from '../../videos/nsfw-policy.type.js'
+
+export interface UserSettingsExportJSON {
+  email: string
+
+  emailPublic: boolean
+  nsfwPolicy: NSFWPolicyType
+
+  autoPlayVideo: boolean
+  autoPlayNextVideo: boolean
+  autoPlayNextVideoPlaylist: boolean
+
+  p2pEnabled: boolean
+
+  videosHistoryEnabled: boolean
+  videoLanguages: string[]
+
+  theme: string
+
+  createdAt: Date
+
+  notificationSettings: UserNotificationSetting
+
+  archiveFiles?: never
+}

packages/models/src/import-export/peertube-export-format/video-export.model.ts

@@ -0,0 +1,103 @@
+import {
+  LiveVideoLatencyModeType,
+  VideoPrivacyType,
+  VideoStateType,
+  VideoStreamingPlaylistType_Type
+} from '../../videos/index.js'
+
+export interface VideoExportJSON {
+  videos: {
+    uuid: string
+
+    createdAt: string
+    updatedAt: string
+    publishedAt: string
+    originallyPublishedAt: string
+
+    name: string
+    category: number
+    licence: number
+    language: string
+    tags: string[]
+
+    privacy: VideoPrivacyType
+    passwords: string[]
+
+    duration: number
+
+    description: string
+    support: string
+
+    isLive: boolean
+    live?: {
+      saveReplay: boolean
+      permanentLive: boolean
+      latencyMode: LiveVideoLatencyModeType
+      streamKey: string
+
+      replaySettings?: {
+        privacy: VideoPrivacyType
+      }
+    }
+
+    url: string
+
+    thumbnailUrl: string
+    previewUrl: string
+
+    views: number
+
+    likes: number
+    dislikes: number
+
+    nsfw: boolean
+
+    commentsEnabled: boolean
+    downloadEnabled: boolean
+
+    channel: {
+      name: string
+    }
+
+    waitTranscoding: boolean
+    state: VideoStateType
+
+    captions: {
+      createdAt: string
+      updatedAt: string
+      language: string
+      filename: string
+      fileUrl: string
+    }[]
+
+    files: VideoFileExportJSON[]
+
+    streamingPlaylists: {
+      type: VideoStreamingPlaylistType_Type
+      playlistUrl: string
+      segmentsSha256Url: string
+      files: VideoFileExportJSON[]
+    }[]
+
+    source?: {
+      filename: string
+    }
+
+    archiveFiles: {
+      videoFile: string | null
+      thumbnail: string | null
+      captions: Record<string, string> // The key is the language code
+    }
+  }[]
+}
+
+// ---------------------------------------------------------------------------
+
+export interface VideoFileExportJSON {
+  resolution: number
+  size: number // Bytes
+  fps: number
+
+  torrentUrl: string
+  fileUrl: string
+}

packages/models/src/import-export/peertube-export-format/video-playlists-export.model.ts

@@ -0,0 +1,34 @@
+import { VideoPlaylistPrivacyType } from '../../videos/playlist/video-playlist-privacy.model.js'
+import { VideoPlaylistType_Type } from '../../videos/playlist/video-playlist-type.model.js'
+
+export interface VideoPlaylistsExportJSON {
+  videoPlaylists: {
+    displayName: string
+    description: string
+    privacy: VideoPlaylistPrivacyType
+    url: string
+    uuid: string
+
+    type: VideoPlaylistType_Type
+
+    channel: {
+      name: string
+    }
+
+    createdAt: string
+    updatedAt: string
+
+    thumbnailUrl: string
+
+    elements: {
+      videoUrl: string
+
+      startTimestamp?: number
+      stopTimestamp?: number
+    }[]
+
+    archiveFiles: {
+      thumbnail: string | null
+    }
+  }[]
+}

packages/models/src/import-export/user-export-request-result.model.ts

@@ -0,0 +1,5 @@
+export interface UserExportRequestResult {
+  export: {
+    id: number
+  }
+}

packages/models/src/import-export/user-export-request.model.ts

@@ -0,0 +1,3 @@
+export interface UserExportRequest {
+  withVideoFiles: boolean
+}

packages/models/src/import-export/user-export-state.enum.ts

@@ -0,0 +1,8 @@
+export const UserExportState = {
+  PENDING: 1,
+  PROCESSING: 2,
+  COMPLETED: 3,
+  ERRORED: 4
+} as const
+
+export type UserExportStateType = typeof UserExportState[keyof typeof UserExportState]

packages/models/src/import-export/user-export.model.ts

@@ -0,0 +1,18 @@
+import { UserExportStateType } from './user-export-state.enum.js'
+
+export interface UserExport {
+  id: number
+
+  state: {
+    id: UserExportStateType
+    label: string
+  }
+
+  // In bytes
+  size: number
+
+  privateDownloadUrl: string
+
+  createdAt: string | Date
+  expiresOn: string | Date
+}

packages/models/src/import-export/user-import-result.model.ts

@@ -0,0 +1,20 @@
+type Summary = {
+  success: number
+  duplicates: number
+  errors: number
+}
+
+export interface UserImportResultSummary {
+  stats: {
+    blocklist: Summary
+    channels: Summary
+    likes: Summary
+    dislikes: Summary
+    following: Summary
+    videoPlaylists: Summary
+    videos: Summary
+
+    account: Summary
+    userSettings: Summary
+  }
+}

packages/models/src/import-export/user-import-state.enum.ts

@@ -0,0 +1,8 @@
+export const UserImportState = {
+  PENDING: 1,
+  PROCESSING: 2,
+  COMPLETED: 3,
+  ERRORED: 4
+} as const
+
+export type UserImportStateType = typeof UserImportState[keyof typeof UserImportState]

packages/models/src/import-export/user-import-upload-result.model.ts

@@ -0,0 +1,5 @@
+export interface UserImportUploadResult {
+  userImport: {
+    id: number
+  }
+}

packages/models/src/import-export/user-import.model.ts

@@ -0,0 +1,10 @@
+import { UserImportStateType } from './user-import-state.enum.js'
+
+export interface UserImport {
+  id: number
+  state: {
+    id: UserImportStateType
+    label: string
+  }
+  createdAt: string
+}

packages/models/src/index.ts

@@ -3,6 +3,7 @@ export * from './actors/index.js'
 export * from './bulk/index.js'
 export * from './common/index.js'
 export * from './custom-markup/index.js'
+export * from './import-export/index.js'
 export * from './feeds/index.js'
 export * from './http/index.js'
 export * from './joinpeertube/index.js'

packages/models/src/plugins/server/server-hook.model.ts

@@ -65,6 +65,8 @@ export const serverFilterHookObject = {
   'filter:api.video.post-import-url.accept.result': true,
   'filter:api.video.post-import-torrent.accept.result': true,
   'filter:api.video.update-file.accept.result': true,
+  // PeerTube >= 6.1
+  'filter:api.video.user-import.accept.result': true,
   // Filter the result of the accept comment (thread or reply) functions
   // If the functions return false then the user cannot post its comment
   'filter:api.video-thread.create.accept.result': true,
@@ -75,6 +77,8 @@ export const serverFilterHookObject = {
   'filter:api.video.import-url.video-attribute.result': true,
   'filter:api.video.import-torrent.video-attribute.result': true,
   'filter:api.video.live.video-attribute.result': true,
+  // PeerTube >= 6.1
+  'filter:api.video.user-import.video-attribute.result': true,
 
   // Filter params/result used to list threads of a specific video
   // (used by the video watch page)

packages/models/src/server/custom-config.model.ts

@@ -193,10 +193,23 @@ export interface CustomConfig {
         enabled: boolean
       }
     }
+
     videoChannelSynchronization: {
       enabled: boolean
       maxPerUser: number
     }
+
+    users: {
+      enabled: boolean
+    }
+  }
+
+  export: {
+    users: {
+      enabled: boolean
+      maxUserVideoQuota: number
+      exportExpiration: number
+    }
   }
 
   trending: {
@@ -260,5 +273,4 @@ export interface CustomConfig {
   storyboards: {
     enabled: boolean
   }
-
 }

packages/models/src/server/debug.model.ts

@@ -9,4 +9,5 @@ export interface SendDebugCommand {
   | 'process-video-viewers'
   | 'process-video-channel-sync-latest'
   | 'process-update-videos-scheduler'
+  | 'remove-expired-user-exports'
 }

packages/models/src/server/job.model.ts

@@ -31,6 +31,8 @@ export type JobType =
   | 'video-transcoding'
   | 'videos-views-stats'
   | 'generate-video-storyboard'
+  | 'create-user-export'
+  | 'import-user-archive'
 
 export interface Job {
   id: number | string
@@ -302,3 +304,15 @@ export interface GenerateStoryboardPayload {
   videoUUID: string
   federate: boolean
 }
+
+// ---------------------------------------------------------------------------
+
+export interface CreateUserExportPayload {
+  userExportId: number
+}
+
+// ---------------------------------------------------------------------------
+
+export interface ImportUserArchivePayload {
+  userImportId: number
+}

packages/models/src/server/server-config.model.ts

@@ -207,9 +207,22 @@ export interface ServerConfig {
         enabled: boolean
       }
     }
+
     videoChannelSynchronization: {
       enabled: boolean
     }
+
+    users: {
+      enabled:boolean
+    }
+  }
+
+  export: {
+    users: {
+      enabled: boolean
+      exportExpiration: number
+      maxUserVideoQuota: number
+    }
   }
 
   autoBlacklist: {

packages/models/src/server/server-error-code.enum.ts

@@ -54,7 +54,9 @@ export const ServerErrorCode = {
   VIDEO_REQUIRES_PASSWORD:'video_requires_password',
   INCORRECT_VIDEO_PASSWORD:'incorrect_video_password',
 
-  VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded'
+  VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded',
+
+  MAX_USER_VIDEO_QUOTA_EXCEEDED_FOR_USER_EXPORT: 'max_user_video_quota_exceeded_for_user_export'
 } as const
 
 /**

packages/models/src/users/user-right.enum.ts

@@ -47,7 +47,10 @@ export const UserRight = {
 
   MANAGE_REGISTRATIONS: 28,
 
-  MANAGE_RUNNERS: 29
+  MANAGE_RUNNERS: 29,
+
+  MANAGE_USER_EXPORTS: 30,
+  MANAGE_USER_IMPORTS: 31
 } as const
 
 export type UserRightType = typeof UserRight[keyof typeof UserRight]

packages/models/src/videos/index.ts

@@ -29,7 +29,6 @@ export * from './video-rate.type.js'
 export * from './video-schedule-update.model.js'
 export * from './video-sort-field.type.js'
 export * from './video-state.enum.js'
-export * from './video-storage.enum.js'
 export * from './video-source.model.js'
 
 export * from './video-streaming-playlist.model.js'

packages/models/src/videos/video-storage.enum.ts

@@ -1,6 +0,0 @@
-export const VideoStorage = {
-  FILE_SYSTEM: 0,
-  OBJECT_STORAGE: 1
-} as const
-
-export type VideoStorageType = typeof VideoStorage[keyof typeof VideoStorage]

packages/node-utils/src/path.ts

@@ -1,4 +1,4 @@
-import { basename, extname, isAbsolute, join, resolve } from 'path'
+import { basename, extname, isAbsolute, join, parse, resolve } from 'path'
 import { fileURLToPath } from 'url'
 
 let rootPath: string
@@ -48,3 +48,15 @@ export function buildAbsoluteFixturePath (path: string, customCIPath = false) {
 
   return join(root(), 'packages', 'tests', 'fixtures', path)
 }
+
+export function getFilenameFromUrl (url: string) {
+  return getFilename(new URL(url).pathname)
+}
+
+export function getFilename (path: string) {
+  return parse(path).base
+}
+
+export function getFilenameWithoutExt (path: string) {
+  return parse(path).name
+}

packages/typescript-utils/src/types.ts

@@ -45,3 +45,5 @@ export type DeepOmitArray<T extends any[], K> = {
 }
 
 export type Unpacked<T> = T extends (infer U)[] ? U : T
+
+export type Awaitable<T> = T | PromiseLike<T>

scripts/i18n/create-custom-files.ts

@@ -6,6 +6,7 @@ import {
   ABUSE_STATES,
   buildLanguages,
   RUNNER_JOB_STATES,
+  USER_EXPORT_STATES,
   USER_REGISTRATION_STATES,
   VIDEO_CATEGORIES,
   VIDEO_CHANNEL_SYNC_STATE,
@@ -14,6 +15,7 @@ import {
   VIDEO_PLAYLIST_PRIVACIES,
   VIDEO_PLAYLIST_TYPES,
   VIDEO_PRIVACIES,
+  USER_IMPORT_STATES,
   VIDEO_STATES
 } from '@peertube/peertube-server/core/initializers/constants.js'
 
@@ -96,6 +98,8 @@ Object.values(VIDEO_CATEGORIES)
   .concat(Object.values(ABUSE_STATES))
   .concat(Object.values(USER_REGISTRATION_STATES))
   .concat(Object.values(RUNNER_JOB_STATES))
+  .concat(Object.values(USER_EXPORT_STATES))
+  .concat(Object.values(USER_IMPORT_STATES))
   .concat([
     'This video does not exist.',
     'We cannot fetch the video. Please try again later.',

server/core/controllers/api/config.ts

@@ -355,6 +355,16 @@ function customConfig (): CustomConfig {
       videoChannelSynchronization: {
         enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
         maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
+      },
+      users: {
+        enabled: CONFIG.IMPORT.USERS.ENABLED
+      }
+    },
+    export: {
+      users: {
+        enabled: CONFIG.EXPORT.USERS.ENABLED,
+        exportExpiration: CONFIG.EXPORT.USERS.EXPORT_EXPIRATION,
+        maxUserVideoQuota: CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA
       }
     },
     trending: {

server/core/controllers/api/index.ts

@@ -50,7 +50,6 @@ apiRouter.use('/custom-pages', customPageRouter)
 apiRouter.use('/blocklist', blocklistRouter)
 apiRouter.use('/runners', runnersRouter)
 
-// apiRouter.use(apiRateLimiter)
 apiRouter.use('/ping', pong)
 apiRouter.use('/*', badRequest)
 

server/core/controllers/api/runners/jobs-files.ts

@@ -9,7 +9,7 @@ import {
   runnerJobGetVideoStudioTaskFileValidator,
   runnerJobGetVideoTranscodingFileValidator
 } from '@server/middlewares/validators/runners/job-files.js'
-import { RunnerJobState, VideoStorage } from '@peertube/peertube-models'
+import { RunnerJobState, FileStorage } from '@peertube/peertube-models'
 
 const lTags = loggerTagsFactory('api', 'runner')
 
@@ -57,7 +57,7 @@ async function getMaxQualityVideoFile (req: express.Request, res: express.Respon
 
   const file = video.getMaxQualityFile()
 
-  if (file.storage === VideoStorage.OBJECT_STORAGE) {
+  if (file.storage === FileStorage.OBJECT_STORAGE) {
     if (file.isHLS()) {
       return proxifyHLS({
         req,

server/core/controllers/api/search/search-videos.ts

@@ -151,7 +151,7 @@ async function searchVideoURI (url: string, res: express.Response) {
       logger.info('Cannot search remote video %s.', url, { err })
     }
   } else {
-    video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url))
+    video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccountAndFiles(url))
   }
 
   return res.json({

server/core/controllers/api/server/debug.ts

@@ -7,6 +7,7 @@ import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-ch
 import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler.js'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
 import { authenticate, ensureUserHasRight } from '../../../middlewares/index.js'
+import { RemoveExpiredUserExportsScheduler } from '@server/lib/schedulers/remove-expired-user-exports-scheduler.js'
 
 const debugRouter = express.Router()
 
@@ -42,6 +43,7 @@ async function runCommand (req: express.Request, res: express.Response) {
 
   const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
     'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
+    'remove-expired-user-exports': () => RemoveExpiredUserExportsScheduler.Instance.execute(),
     'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
     'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
     'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),

server/core/controllers/api/server/server-blocklist.ts

@@ -1,9 +1,7 @@
 import 'multer'
 import express from 'express'
 import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
-import { logger } from '@server/helpers/logger.js'
 import { getServerActor } from '@server/models/application/application.js'
-import { UserNotificationModel } from '@server/models/user/user-notification.js'
 import { getFormattedObjects } from '../../../helpers/utils.js'
 import {
   addAccountInBlocklist,
@@ -105,15 +103,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
   const accountToBlock = res.locals.account
 
-  await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id)
+  await addAccountInBlocklist({ byAccountId: serverActor.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: null })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: accountToBlock.id,
-    type: 'account',
-    forUserId: null // For all users
-  }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockAccount (req: express.Request, res: express.Response) {
@@ -121,7 +113,7 @@ async function unblockAccount (req: express.Request, res: express.Response) {
 
   await removeAccountFromBlocklist(accountBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function listBlockedServers (req: express.Request, res: express.Response) {
@@ -142,15 +134,13 @@ async function blockServer (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
   const serverToBlock = res.locals.server
 
-  await addServerInBlocklist(serverActor.Account.id, serverToBlock.id)
+  await addServerInBlocklist({
+    byAccountId: serverActor.Account.id,
+    targetServerId: serverToBlock.id,
+    removeNotificationOfUserId: null
+  })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: serverToBlock.id,
-    type: 'server',
-    forUserId: null // For all users
-  }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockServer (req: express.Request, res: express.Response) {
@@ -158,5 +148,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
 
   await removeServerFromBlocklist(serverBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/index.ts

@@ -47,6 +47,8 @@ import { mySubscriptionsRouter } from './my-subscriptions.js'
 import { myVideoPlaylistsRouter } from './my-video-playlists.js'
 import { registrationsRouter } from './registrations.js'
 import { twoFactorRouter } from './two-factor.js'
+import { userExportsRouter } from './user-exports.js'
+import { userImportRouter } from './user-imports.js'
 
 const auditLogger = auditLoggerFactory('users')
 
@@ -55,6 +57,8 @@ const usersRouter = express.Router()
 usersRouter.use(apiRateLimiter)
 
 usersRouter.use('/', emailVerificationRouter)
+usersRouter.use('/', userExportsRouter)
+usersRouter.use('/', userImportRouter)
 usersRouter.use('/', registrationsRouter)
 usersRouter.use('/', twoFactorRouter)
 usersRouter.use('/', tokensRouter)

server/core/controllers/api/users/me.ts

@@ -262,11 +262,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response) {
 
   const userAccount = await AccountModel.load(user.Account.id)
 
-  const avatars = await updateLocalActorImageFiles(
-    userAccount,
-    avatarPhysicalFile,
-    ActorImageType.AVATAR
-  )
+  const avatars = await updateLocalActorImageFiles({
+    accountOrChannel: userAccount,
+    imagePhysicalFile: avatarPhysicalFile,
+    type: ActorImageType.AVATAR,
+    sendActorUpdate: true
+  })
 
   return res.json({
     avatars: avatars.map(avatar => avatar.toFormattedJSON())

server/core/controllers/api/users/my-blocklist.ts

@@ -1,8 +1,6 @@
 import 'multer'
 import express from 'express'
 import { HttpStatusCode } from '@peertube/peertube-models'
-import { logger } from '@server/helpers/logger.js'
-import { UserNotificationModel } from '@server/models/user/user-notification.js'
 import { getFormattedObjects } from '../../../helpers/utils.js'
 import {
   addAccountInBlocklist,
@@ -97,15 +95,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
   const user = res.locals.oauth.token.User
   const accountToBlock = res.locals.account
 
-  await addAccountInBlocklist(user.Account.id, accountToBlock.id)
+  await addAccountInBlocklist({ byAccountId: user.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: user.id })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: accountToBlock.id,
-    type: 'account',
-    forUserId: user.id
-  }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockAccount (req: express.Request, res: express.Response) {
@@ -134,15 +126,13 @@ async function blockServer (req: express.Request, res: express.Response) {
   const user = res.locals.oauth.token.User
   const serverToBlock = res.locals.server
 
-  await addServerInBlocklist(user.Account.id, serverToBlock.id)
+  await addServerInBlocklist({
+    byAccountId: user.Account.id,
+    targetServerId: serverToBlock.id,
+    removeNotificationOfUserId: user.id
+  })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: serverToBlock.id,
-    type: 'server',
-    forUserId: user.id
-  }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockServer (req: express.Request, res: express.Response) {
@@ -150,5 +140,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
 
   await removeServerFromBlocklist(serverBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/my-notifications.ts

@@ -16,7 +16,7 @@ import {
   listUserNotificationsValidator,
   markAsReadUserNotificationsValidator,
   updateNotificationSettingsValidator
-} from '../../../middlewares/validators/user-notifications.js'
+} from '../../../middlewares/validators/users/user-notifications.js'
 import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting.js'
 import { meRouter } from './me.js'
 
@@ -59,12 +59,6 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
   const user = res.locals.oauth.token.User
   const body = req.body as UserNotificationSetting
 
-  const query = {
-    where: {
-      userId: user.id
-    }
-  }
-
   const values: UserNotificationSetting = {
     newVideoFromSubscription: body.newVideoFromSubscription,
     newCommentOnMyVideo: body.newCommentOnMyVideo,
@@ -85,9 +79,9 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
     myVideoStudioEditionFinished: body.myVideoStudioEditionFinished
   }
 
-  await UserNotificationSettingModel.update(values, query)
+  await UserNotificationSettingModel.updateUserSettings(values, user.id)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function listUserNotifications (req: express.Request, res: express.Response) {
@@ -103,7 +97,7 @@ async function markAsReadUserNotifications (req: express.Request, res: express.R
 
   await UserNotificationModel.markAsRead(user.id, req.body.ids)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) {
@@ -111,5 +105,5 @@ async function markAsReadAllUserNotifications (req: express.Request, res: expres
 
   await UserNotificationModel.markAllAsRead(user.id)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/user-exports.ts

@@ -0,0 +1,100 @@
+import express from 'express'
+import { FileStorage, HttpStatusCode, UserExportRequest, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
+import {
+  apiRateLimiter,
+  asyncMiddleware,
+  authenticate,
+  userExportDeleteValidator,
+  userExportRequestValidator,
+  userExportsListValidator
+} from '../../../middlewares/index.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { getFormattedObjects } from '@server/helpers/utils.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
+import { CONFIG } from '@server/initializers/config.js'
+
+const userExportsRouter = express.Router()
+
+userExportsRouter.use(apiRateLimiter)
+
+userExportsRouter.post('/:userId/exports/request',
+  authenticate,
+  asyncMiddleware(userExportRequestValidator),
+  asyncMiddleware(requestExport)
+)
+
+userExportsRouter.get('/:userId/exports',
+  authenticate,
+  asyncMiddleware(userExportsListValidator),
+  asyncMiddleware(listUserExports)
+)
+
+userExportsRouter.delete('/:userId/exports/:id',
+  authenticate,
+  asyncMiddleware(userExportDeleteValidator),
+  asyncMiddleware(deleteUserExport)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  userExportsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function requestExport (req: express.Request, res: express.Response) {
+  const body = req.body as UserExportRequest
+
+  const exportModel = new UserExportModel({
+    state: UserExportState.PENDING,
+    withVideoFiles: body.withVideoFiles,
+
+    storage: CONFIG.OBJECT_STORAGE.ENABLED
+      ? FileStorage.OBJECT_STORAGE
+      : FileStorage.FILE_SYSTEM,
+
+    userId: res.locals.user.id,
+    createdAt: new Date()
+  })
+  exportModel.generateAndSetFilename()
+
+  await sequelizeTypescript.transaction(async transaction => {
+    await exportModel.save({ transaction })
+  })
+
+  await JobQueue.Instance.createJob({ type: 'create-user-export', payload: { userExportId: exportModel.id } })
+
+  return res.json({
+    export: {
+      id: exportModel.id
+    }
+  } as UserExportRequestResult)
+}
+
+async function listUserExports (req: express.Request, res: express.Response) {
+  const resultList = await UserExportModel.listForApi({
+    start: req.query.start,
+    count: req.query.count,
+    user: res.locals.user
+  })
+
+  return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
+async function deleteUserExport (req: express.Request, res: express.Response) {
+  const userExport = res.locals.userExport
+
+  await sequelizeTypescript.transaction(async transaction => {
+    await userExport.reload({ transaction })
+
+    if (!userExport.canBeSafelyRemoved()) {
+      return res.sendStatus(HttpStatusCode.CONFLICT_409)
+    }
+
+    await userExport.destroy({ transaction })
+  })
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}

server/core/controllers/api/users/user-imports.ts

@@ -0,0 +1,90 @@
+import express from 'express'
+import {
+  apiRateLimiter,
+  asyncMiddleware,
+  authenticate
+} from '../../../middlewares/index.js'
+import { uploadx } from '@server/lib/uploadx.js'
+import {
+  getLatestImportStatusValidator,
+  userImportRequestResumableInitValidator,
+  userImportRequestResumableValidator
+} from '@server/middlewares/validators/users/user-import.js'
+import { HttpStatusCode, UserImportState, UserImportUploadResult } from '@peertube/peertube-models'
+import { logger } from '@server/helpers/logger.js'
+import { UserImportModel } from '@server/models/user/user-import.js'
+import { getFSUserImportFilePath } from '@server/lib/paths.js'
+import { move } from 'fs-extra/esm'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
+import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
+
+const userImportRouter = express.Router()
+
+userImportRouter.use(apiRateLimiter)
+
+userImportRouter.post('/:userId/imports/import-resumable',
+  authenticate,
+  asyncMiddleware(userImportRequestResumableInitValidator),
+  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+)
+
+userImportRouter.delete('/:userId/imports/import-resumable',
+  authenticate,
+  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+)
+
+userImportRouter.put('/:userId/imports/import-resumable',
+  authenticate,
+  uploadx.upload, // uploadx doesn't next() before the file upload completes
+  asyncMiddleware(userImportRequestResumableValidator),
+  asyncMiddleware(addUserImportResumable)
+)
+
+userImportRouter.get('/:userId/imports/latest',
+  authenticate,
+  asyncMiddleware(getLatestImportStatusValidator),
+  asyncMiddleware(getLatestImport)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  userImportRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function addUserImportResumable (req: express.Request, res: express.Response) {
+  const file = res.locals.importUserFileResumable
+  const user = res.locals.user
+
+  // Move import
+  const userImport = new UserImportModel({
+    state: UserImportState.PENDING,
+    userId: user.id,
+    createdAt: new Date()
+  })
+  userImport.generateAndSetFilename()
+
+  await move(file.path, getFSUserImportFilePath(userImport))
+
+  await saveInTransactionWithRetries(userImport)
+
+  // Create job
+  await JobQueue.Instance.createJob({ type: 'import-user-archive', payload: { userImportId: userImport.id } })
+
+  logger.info('User import request job created for user ' + user.username)
+
+  return res.json({
+    userImport: {
+      id: userImport.id
+    }
+  } as UserImportUploadResult)
+}
+
+async function getLatestImport (req: express.Request, res: express.Response) {
+  const userImport = await UserImportModel.loadLatestByUserId(res.locals.user.id)
+  if (!userImport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+
+  return res.json(userImport.toFormattedJSON())
+}

server/core/controllers/api/video-channel.ts

@@ -213,7 +213,12 @@ async function updateVideoChannelBanner (req: express.Request, res: express.Resp
   const videoChannel = res.locals.videoChannel
   const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
 
-  const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER)
+  const banners = await updateLocalActorImageFiles({
+    accountOrChannel: videoChannel,
+    imagePhysicalFile: bannerPhysicalFile,
+    type: ActorImageType.BANNER,
+    sendActorUpdate: true
+  })
 
   auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
 
@@ -227,7 +232,13 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
   const videoChannel = res.locals.videoChannel
   const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
 
-  const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR)
+  const avatars = await updateLocalActorImageFiles({
+    accountOrChannel: videoChannel,
+    imagePhysicalFile: avatarPhysicalFile,
+    type: ActorImageType.AVATAR,
+    sendActorUpdate: true
+  })
+
   auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
 
   return res.json({

server/core/controllers/api/video-playlist.ts

@@ -192,7 +192,6 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
     const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull
 
     if (thumbnailModel) {
-      thumbnailModel.automaticallyGenerated = false
       await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t)
     }
 

server/core/controllers/api/videos/rate.ts

@@ -1,12 +1,8 @@
 import express from 'express'
 import { HttpStatusCode, UserVideoRateUpdate } from '@peertube/peertube-models'
 import { logger } from '../../../helpers/logger.js'
-import { VIDEO_RATE_TYPES } from '../../../initializers/constants.js'
-import { sequelizeTypescript } from '../../../initializers/database.js'
-import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates.js'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares/index.js'
-import { AccountModel } from '../../../models/account/account.js'
-import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
+import { userRateVideo } from '@server/lib/rate.js'
 
 const rateVideoRouter = express.Router()
 
@@ -25,63 +21,16 @@ export {
 // ---------------------------------------------------------------------------
 
 async function rateVideo (req: express.Request, res: express.Response) {
-  const body: UserVideoRateUpdate = req.body
-  const rateType = body.rating
-  const videoInstance = res.locals.videoAll
-  const userAccount = res.locals.oauth.token.User.Account
+  const user = res.locals.oauth.token.User
+  const video = res.locals.videoAll
 
-  await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    const accountInstance = await AccountModel.load(userAccount.id, t)
-    const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t)
-
-    // Same rate, nothing do to
-    if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return
-
-    let likesToIncrement = 0
-    let dislikesToIncrement = 0
-
-    if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++
-    else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++
-
-    // There was a previous rate, update it
-    if (previousRate) {
-      // We will remove the previous rate, so we will need to update the video count attribute
-      if (previousRate.type === 'like') likesToIncrement--
-      else if (previousRate.type === 'dislike') dislikesToIncrement--
-
-      if (rateType === 'none') { // Destroy previous rate
-        await previousRate.destroy(sequelizeOptions)
-      } else { // Update previous rate
-        previousRate.type = rateType
-        previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
-        await previousRate.save(sequelizeOptions)
-      }
-    } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate
-      const query = {
-        accountId: accountInstance.id,
-        videoId: videoInstance.id,
-        type: rateType,
-        url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
-      }
-
-      await AccountVideoRateModel.create(query, sequelizeOptions)
-    }
-
-    const incrementQuery = {
-      likes: likesToIncrement,
-      dislikes: dislikesToIncrement
-    }
-
-    await videoInstance.increment(incrementQuery, sequelizeOptions)
-
-    await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t)
-
-    logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
+  await userRateVideo({
+    account: user.Account,
+    rateType: (req.body as UserVideoRateUpdate).rating,
+    video
   })
 
-  return res.type('json')
-            .status(HttpStatusCode.NO_CONTENT_204)
-            .end()
+  logger.info('Account video rate for video %s of account %s updated.', video.name, user.username)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/videos/source.ts

@@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
 import { uploadx } from '@server/lib/uploadx.js'
-import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js'
+import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { buildNewFile } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'

server/core/controllers/api/videos/update.ts

@@ -6,7 +6,7 @@ import { exists } from '@server/helpers/custom-validators/misc.js'
 import { changeVideoChannelShare } from '@server/lib/activitypub/share.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { setVideoPrivacy } from '@server/lib/video-privacy.js'
-import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
+import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
 import { VideoPasswordModel } from '@server/models/video/video-password.js'
 import { FilteredModelAttributes } from '@server/types/index.js'
@@ -23,6 +23,7 @@ import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosU
 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js'
 import { VideoModel } from '../../../models/video/video.js'
 import { replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
+import { addVideoJobsAfterUpdate } from '@server/lib/video-jobs.js'
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')

server/core/controllers/api/videos/upload.ts

@@ -3,14 +3,10 @@ import { move } from 'fs-extra/esm'
 import { basename } from 'path'
 import { getResumableUploadPath } from '@server/helpers/upload.js'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
-import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
 import { Redis } from '@server/lib/redis.js'
 import { uploadx } from '@server/lib/uploadx.js'
 import {
-  buildLocalVideoFromReq,
-  buildMoveJob,
-  buildStoryboardJobIfNeeded,
-  buildVideoThumbnailsFromReq,
+  buildLocalVideoFromReq, buildVideoThumbnailsFromReq,
   setVideoTags
 } from '@server/lib/video.js'
 import { buildNewFile } from '@server/lib/video-file.js'
@@ -21,7 +17,7 @@ import { VideoPasswordModel } from '@server/models/video/video-password.js'
 import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
 import { uuidToShort } from '@peertube/peertube-node-utils'
-import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@peertube/peertube-models'
+import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy } from '@peertube/peertube-models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
 import { createReqFiles } from '../../../helpers/express-utils.js'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
@@ -43,6 +39,7 @@ import { VideoModel } from '../../../models/video/video.js'
 import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
 import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
 import { FfprobeData } from 'fluent-ffmpeg'
+import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js'
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')
@@ -230,7 +227,7 @@ async function addVideo (options: {
   // Channel has a new content, set as updated
   await videoCreated.VideoChannel.setAsUpdated()
 
-  addVideoJobsAfterUpload(videoCreated, videoFile)
+  addVideoJobsAfterCreation({ video: videoCreated, videoFile })
     .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 
   Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
@@ -244,55 +241,6 @@ async function addVideo (options: {
   }
 }
 
-async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
-  const jobs: (CreateJobArgument & CreateJobOptions)[] = [
-    {
-      type: 'manage-video-torrent' as 'manage-video-torrent',
-      payload: {
-        videoId: video.id,
-        videoFileId: videoFile.id,
-        action: 'create'
-      }
-    },
-
-    buildStoryboardJobIfNeeded({ video, federate: false }),
-
-    {
-      type: 'notify',
-      payload: {
-        action: 'new-video',
-        videoUUID: video.uuid
-      }
-    },
-
-    {
-      type: 'federate-video' as 'federate-video',
-      payload: {
-        videoUUID: video.uuid,
-        isNewVideoForFederation: true
-      }
-    }
-  ]
-
-  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
-    jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
-  }
-
-  if (video.state === VideoState.TO_TRANSCODE) {
-    jobs.push({
-      type: 'transcoding-job-builder' as 'transcoding-job-builder',
-      payload: {
-        videoUUID: video.uuid,
-        optimizeJob: {
-          isNewVideo: true
-        }
-      }
-    })
-  }
-
-  return JobQueue.Instance.createSequentialJobFlow(...jobs)
-}
-
 async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
   await Redis.Instance.deleteUploadSession(req.query.upload_id)
 

server/core/controllers/download.ts

@@ -2,14 +2,30 @@ import cors from 'cors'
 import express from 'express'
 import { logger } from '@server/helpers/logger.js'
 import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js'
-import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js'
+import {
+  generateHLSFilePresignedUrl,
+  generateUserExportPresignedUrl,
+  generateWebVideoPresignedUrl
+} from '@server/lib/object-storage/index.js'
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
+import {
+  MStreamingPlaylist,
+  MStreamingPlaylistVideo,
+  MUserExport,
+  MVideo,
+  MVideoFile,
+  MVideoFullLight
+} from '@server/types/models/index.js'
 import { forceNumber } from '@peertube/peertube-core-utils'
-import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
+import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
 import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js'
-import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares/index.js'
+import {
+  asyncMiddleware, optionalAuthenticate,
+  userExportDownloadValidator,
+  videosDownloadValidator
+} from '../middlewares/index.js'
+import { getFSUserExportFilePath } from '@server/lib/paths.js'
 
 const downloadRouter = express.Router()
 
@@ -34,6 +50,12 @@ downloadRouter.use(
   asyncMiddleware(downloadHLSVideoFile)
 )
 
+downloadRouter.use(
+  STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename',
+  asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication
+  asyncMiddleware(downloadUserExport)
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -99,8 +121,8 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
   const videoName = video.name.replace(/[/\\]/g, '_')
   const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}`
 
-  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
-    return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename })
+  if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectVideoDownloadToObjectStorage({ res, video, file: videoFile, downloadFilename })
   }
 
   await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@@ -140,8 +162,8 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
   const videoName = video.name.replace(/\//g, '_')
   const downloadFilename = `${videoName}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
 
-  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
-    return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename })
+  if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectVideoDownloadToObjectStorage({ res, video, streamingPlaylist, file: videoFile, downloadFilename })
   }
 
   await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@@ -149,6 +171,21 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
   })
 }
 
+function downloadUserExport (req: express.Request, res: express.Response) {
+  const userExport = res.locals.userExport
+
+  const downloadFilename = userExport.filename
+
+  if (userExport.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectUserExportToObjectStorage({ res, userExport, downloadFilename })
+  }
+
+  res.download(getFSUserExportFilePath(userExport), downloadFilename)
+  return Promise.resolve()
+}
+
+// ---------------------------------------------------------------------------
+
 function getVideoFile (req: express.Request, files: MVideoFile[]) {
   const resolution = forceNumber(req.params.resolution)
   return files.find(f => f.resolution === resolution)
@@ -194,8 +231,7 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
   return true
 }
 
-async function redirectToObjectStorage (options: {
-  req: express.Request
+async function redirectVideoDownloadToObjectStorage (options: {
   res: express.Response
   video: MVideo
   file: MVideoFile
@@ -212,3 +248,17 @@ async function redirectToObjectStorage (options: {
 
   return res.redirect(url)
 }
+
+async function redirectUserExportToObjectStorage (options: {
+  res: express.Response
+  downloadFilename: string
+  userExport: MUserExport
+}) {
+  const { res, downloadFilename, userExport } = options
+
+  const url = await generateUserExportPresignedUrl({ userExport, downloadFilename })
+
+  logger.debug('Generating pre-signed URL %s for user export %s', url, userExport.filename)
+
+  return res.redirect(url)
+}

server/core/helpers/captions-utils.ts

@@ -1,14 +1,11 @@
 import { createReadStream, createWriteStream } from 'fs'
 import { move, remove } from 'fs-extra/esm'
-import { join } from 'path'
 import { Transform } from 'stream'
 import { MVideoCaption } from '@server/types/models/index.js'
-import { CONFIG } from '../initializers/config.js'
 import { pipelinePromise } from './core-utils.js'
 
-async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) {
-  const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR
-  const destination = join(videoCaptionsDir, videoCaption.filename)
+async function moveAndProcessCaptionFile (physicalFile: { filename?: string, path: string }, videoCaption: MVideoCaption) {
+  const destination = videoCaption.getFSPath()
 
   // Convert this srt file to vtt
   if (physicalFile.path.endsWith('.srt')) {
@@ -19,7 +16,7 @@ async function moveAndProcessCaptionFile (physicalFile: { filename: string, path
   }
 
   // This is important in case if there is another attempt in the retry process
-  physicalFile.filename = videoCaption.filename
+  if (physicalFile.filename) physicalFile.filename = videoCaption.filename
   physicalFile.path = destination
 }
 

server/core/helpers/unzip.ts

@@ -0,0 +1,55 @@
+import { createWriteStream } from 'fs'
+import { ensureDir } from 'fs-extra/esm'
+import { dirname, join } from 'path'
+import { pipeline } from 'stream'
+import * as yauzl from 'yauzl'
+import { logger, loggerTagsFactory } from './logger.js'
+
+const lTags = loggerTagsFactory('unzip')
+
+export async function unzip (source: string, destination: string) {
+  await ensureDir(destination)
+
+  logger.info(`Unzip ${source} to ${destination}`, lTags())
+
+  return new Promise<void>((res, rej) => {
+    yauzl.open(source, { lazyEntries: true }, (err, zipFile) => {
+      if (err) return rej(err)
+
+      zipFile.readEntry()
+
+      zipFile.on('entry', async entry => {
+        const entryPath = join(destination, entry.fileName)
+
+        try {
+          if (/\/$/.test(entry.fileName)) {
+            await ensureDir(entryPath)
+            logger.debug(`Creating directory from zip ${entryPath}`, lTags())
+
+            zipFile.readEntry()
+            return
+          }
+
+          await ensureDir(dirname(entryPath))
+        } catch (err) {
+          return rej(err)
+        }
+
+        zipFile.openReadStream(entry, (readErr, readStream) => {
+          if (readErr) return rej(readErr)
+
+          logger.debug(`Creating file from zip ${entryPath}`, lTags())
+
+          const writeStream = createWriteStream(entryPath)
+          writeStream.on('close', () => zipFile.readEntry())
+
+          pipeline(readStream, writeStream, pipelineErr => {
+            if (pipelineErr) return rej(pipelineErr)
+          })
+        })
+      })
+
+      zipFile.on('end', () => res())
+    })
+  })
+}

server/core/initializers/config.ts

@@ -153,6 +153,11 @@ const CONFIG = {
       BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'),
       PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'),
       BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url')
+    },
+    USER_EXPORTS: {
+      BUCKET_NAME: config.get<string>('object_storage.user_exports.bucket_name'),
+      PREFIX: config.get<string>('object_storage.user_exports.prefix'),
+      BASE_URL: config.get<string>('object_storage.user_exports.base_url')
     }
   },
   WEBSERVER: {
@@ -511,6 +516,16 @@ const CONFIG = {
       get FULL_SYNC_VIDEOS_LIMIT () {
         return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
       }
+    },
+    USERS: {
+      get ENABLED () { return config.get<boolean>('import.users.enabled') }
+    }
+  },
+  EXPORT: {
+    USERS: {
+      get ENABLED () { return config.get<boolean>('export.users.enabled') },
+      get MAX_USER_VIDEO_QUOTA () { return parseBytes(config.get<string>('export.users.max_user_video_quota')) },
+      get EXPORT_EXPIRATION () { return parseDurationToMs(config.get<string>('export.users.export_expiration')) }
     }
   },
   AUTO_BLACKLIST: {

server/core/initializers/constants.ts

@@ -10,6 +10,10 @@ import {
   NSFWPolicyType,
   RunnerJobState,
   RunnerJobStateType,
+  UserExportState,
+  UserExportStateType,
+  UserImportState,
+  UserImportStateType,
   UserRegistrationState,
   UserRegistrationStateType,
   VideoChannelSyncState,
@@ -41,7 +45,7 @@ import { cpus } from 'os'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 805
+const LAST_MIGRATION_VERSION = 815
 
 // ---------------------------------------------------------------------------
 
@@ -191,7 +195,9 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'transcoding-job-builder': 1,
   'generate-video-storyboard': 1,
   'notify': 1,
-  'federate-video': 1
+  'federate-video': 1,
+  'create-user-export': 1,
+  'import-user-archive': 1
 }
 // Excluded keys are jobs that can be configured by admins
 const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
@@ -217,7 +223,9 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'transcoding-job-builder': 1,
   'generate-video-storyboard': 1,
   'notify': 5,
-  'federate-video': 3
+  'federate-video': 3,
+  'create-user-export': 1,
+  'import-user-archive': 1
 }
 const JOB_TTL: { [id in JobType]: number } = {
   'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@@ -244,7 +252,9 @@ const JOB_TTL: { [id in JobType]: number } = {
   'after-video-channel-import': 60000 * 5, // 5 minutes
   'transcoding-job-builder': 60000, // 1 minute
   'notify': 60000 * 5, // 5 minutes
-  'federate-video': 60000 * 5 // 5 minutes
+  'federate-video': 60000 * 5, // 5 minutes,
+  'create-user-export': 60000 * 60 * 24, // 24 hours
+  'import-user-archive': 60000 * 60 * 24 // 24 hours
 }
 const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
   'videos-views-stats': {
@@ -313,6 +323,7 @@ const SCHEDULER_INTERVALS_MS = {
   AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day
   REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
   REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
+  REMOVE_EXPIRED_USER_EXPORTS: 1000 * 3600, // 1 hour
   UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
   REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
   CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
@@ -503,6 +514,10 @@ const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
   DISLIKE: 'dislike'
 }
 
+const USER_IMPORT = {
+  MAX_PLAYLIST_ELEMENTS: 1000
+}
+
 const FFMPEG_NICE = {
   // parent process defaults to niceness = 0
   // reminder: lower = higher priority, max value is 19, lowest is -20
@@ -618,6 +633,20 @@ const RUNNER_JOB_STATES: { [ id in RunnerJobStateType ]: string } = {
   [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
 }
 
+const USER_EXPORT_STATES: { [ id in UserExportStateType ]: string } = {
+  [UserExportState.PENDING]: 'Pending',
+  [UserExportState.PROCESSING]: 'Processing',
+  [UserExportState.COMPLETED]: 'Completed',
+  [UserExportState.ERRORED]: 'Failed'
+}
+
+const USER_IMPORT_STATES: { [ id in UserImportStateType ]: string } = {
+  [UserImportState.PENDING]: 'Pending',
+  [UserImportState.PROCESSING]: 'Processing',
+  [UserImportState.COMPLETED]: 'Completed',
+  [UserImportState.ERRORED]: 'Failed'
+}
+
 const MIMETYPES = {
   AUDIO: {
     MIMETYPE_EXT: {
@@ -773,6 +802,7 @@ const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
 const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
 
 const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes
+let JWT_TOKEN_USER_EXPORT_FILE_LIFETIME = '15 minutes'
 
 const EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
 
@@ -807,7 +837,8 @@ const STATIC_PATHS = {
 const STATIC_DOWNLOAD_PATHS = {
   TORRENTS: '/download/torrents/',
   VIDEOS: '/download/videos/',
-  HLS_VIDEOS: '/download/streaming-playlists/hls/videos/'
+  HLS_VIDEOS: '/download/streaming-playlists/hls/videos/',
+  USER_EXPORT: '/download/user-export/'
 }
 const LAZY_STATIC_PATHS = {
   THUMBNAILS: '/lazy-static/thumbnails/',
@@ -1125,6 +1156,8 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
     VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1
 
     RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL = 2000
+
+    JWT_TOKEN_USER_EXPORT_FILE_LIFETIME = '2 seconds'
   }
 }
 
@@ -1168,6 +1201,8 @@ export {
   DIRECTORIES,
   RESUMABLE_UPLOAD_SESSION_LIFETIME,
   RUNNER_JOB_STATES,
+  USER_EXPORT_STATES,
+  USER_IMPORT_STATES,
   P2P_MEDIA_LOADER_PEER_VERSION,
   STORYBOARD,
   ACTOR_IMAGES_SIZE,
@@ -1187,6 +1222,7 @@ export {
   STATS_TIMESERIE,
   BROADCAST_CONCURRENCY,
   AUDIT_LOG_FILENAME,
+  USER_IMPORT,
   PAGINATION,
   ACTOR_FOLLOW_SCORE,
   PREVIEWS_SIZE,
@@ -1195,6 +1231,7 @@ export {
   DEFAULT_USER_THEME_NAME,
   SERVER_ACTOR_NAME,
   TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
+  JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
   PLUGIN_GLOBAL_CSS_FILE_NAME,
   PLUGIN_GLOBAL_CSS_PATH,
   PRIVATE_RSA_KEY_SIZE,

server/core/initializers/database.ts

@@ -60,6 +60,8 @@ import { VideoModel } from '../models/video/video.js'
 import { VideoViewModel } from '../models/view/video-view.js'
 import { CONFIG } from './config.js'
 import { VideoChapterModel } from '@server/models/video/video-chapter.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { UserImportModel } from '@server/models/user/user-import.js'
 
 pg.defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -165,6 +167,7 @@ async function initDatabaseModels (silent: boolean) {
     VideoTrackerModel,
     PluginModel,
     ActorCustomPageModel,
+    UserImportModel,
     VideoJobInfoModel,
     VideoChannelSyncModel,
     UserRegistrationModel,
@@ -172,7 +175,8 @@ async function initDatabaseModels (silent: boolean) {
     RunnerRegistrationTokenModel,
     RunnerModel,
     RunnerJobModel,
-    StoryboardModel
+    StoryboardModel,
+    UserExportModel
   ])
 
   // Check extensions exist in the database

server/core/initializers/migrations/0660-object-storage.ts

@@ -1,5 +1,5 @@
 import * as Sequelize from 'sequelize'
-import { VideoStorage } from '@peertube/peertube-models'
+import { FileStorage } from '@peertube/peertube-models'
 
 async function up (utils: {
   transaction: Sequelize.Transaction
@@ -27,7 +27,7 @@ async function up (utils: {
     await utils.queryInterface.addColumn('videoFile', 'storage', {
       type: Sequelize.INTEGER,
       allowNull: true,
-      defaultValue: VideoStorage.FILE_SYSTEM
+      defaultValue: FileStorage.FILE_SYSTEM
     })
     await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false, defaultValue: null })
   }
@@ -36,7 +36,7 @@ async function up (utils: {
     await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', {
       type: Sequelize.INTEGER,
       allowNull: true,
-      defaultValue: VideoStorage.FILE_SYSTEM
+      defaultValue: FileStorage.FILE_SYSTEM
     })
     await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', {
       type: Sequelize.INTEGER,

server/core/initializers/migrations/0810-user-export.ts

@@ -0,0 +1,33 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  const query = `
+  CREATE TABLE IF NOT EXISTS "userExport" (
+    "id" SERIAL,
+    "filename" VARCHAR(255),
+    "withVideoFiles" BOOLEAN NOT NULL,
+    "state" INTEGER NOT NULL,
+    "error" TEXT,
+    "size" INTEGER,
+    "storage" INTEGER NOT NULL,
+    "userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+    PRIMARY KEY ("id")
+  );`
+
+  await utils.sequelize.query(query, { transaction: utils.transaction })
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/core/initializers/migrations/0815-user-import.ts

@@ -0,0 +1,31 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  const query = `
+  CREATE TABLE IF NOT EXISTS "userImport" (
+    "id" SERIAL,
+    "filename" VARCHAR(255),
+    "state" INTEGER NOT NULL,
+    "error" TEXT,
+    "resultSummary" JSONB,
+    "userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+    PRIMARY KEY ("id")
+  );;`
+
+  await utils.sequelize.query(query, { transaction: utils.transaction })
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/core/lib/activitypub/collection.ts

@@ -7,7 +7,7 @@ import { forceNumber } from '@peertube/peertube-core-utils'
 
 type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
 
-async function activityPubCollectionPagination (
+export async function activityPubCollectionPagination (
   baseUrl: string,
   handler: ActivityPubCollectionPaginationHandler,
   page?: any,
@@ -56,8 +56,11 @@ async function activityPubCollectionPagination (
   }
 }
 
-// ---------------------------------------------------------------------------
-
-export {
-  activityPubCollectionPagination
+export function activityPubCollection <T> (baseUrl: string, items: T[]) {
+  return {
+    id: baseUrl,
+    type: 'OrderedCollection' as 'OrderedCollection',
+    totalItems: items.length,
+    orderedItems: items
+  }
 }

server/core/lib/activitypub/process/process-delete.ts

@@ -51,7 +51,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDelete
   }
 
   {
-    const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl)
+    const videoInstance = await VideoModel.loadByUrlAndPopulateAccountAndFiles(objectUrl)
     if (videoInstance) {
       if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)
 

server/core/lib/activitypub/process/process-flag.ts

@@ -47,7 +47,7 @@ async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature)
       logger.debug('Reporting remote abuse for object %s.', uri)
 
       await sequelizeTypescript.transaction(async t => {
-        const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t)
+        const video = await VideoModel.loadByUrlAndPopulateAccountAndFiles(uri, t)
         let videoComment: MCommentOwnerVideo
         let flaggedAccount: MAccountDefault
 

server/core/lib/activitypub/videos/refresh.ts

@@ -18,7 +18,7 @@ async function refreshVideoIfNeeded (options: {
   // We need more attributes if the argument video was fetched with not enough joints
   const video = options.fetchedType === 'all'
     ? options.video as MVideoAccountLightBlacklistAllFiles
-    : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
+    : await VideoModel.loadByUrlAndPopulateAccountAndFiles(options.video.url)
 
   const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url)
 

server/core/lib/blocklist.ts

@@ -3,23 +3,51 @@ import { getServerActor } from '@server/models/application/application.js'
 import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models/index.js'
 import { AccountBlocklistModel } from '../models/account/account-blocklist.js'
 import { ServerBlocklistModel } from '../models/server/server-blocklist.js'
+import { UserNotificationModel } from '@server/models/user/user-notification.js'
+import { logger } from '@server/helpers/logger.js'
 
-function addAccountInBlocklist (byAccountId: number, targetAccountId: number) {
-  return sequelizeTypescript.transaction(async t => {
+async function addAccountInBlocklist (options: {
+  byAccountId: number
+  targetAccountId: number
+
+  removeNotificationOfUserId: number | null // If blocked by a user
+}) {
+  const { byAccountId, targetAccountId, removeNotificationOfUserId } = options
+
+  await sequelizeTypescript.transaction(async t => {
     return AccountBlocklistModel.upsert({
       accountId: byAccountId,
       targetAccountId
     }, { transaction: t })
   })
+
+  UserNotificationModel.removeNotificationsOf({
+    id: targetAccountId,
+    type: 'account',
+    forUserId: removeNotificationOfUserId
+  }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
 }
 
-function addServerInBlocklist (byAccountId: number, targetServerId: number) {
-  return sequelizeTypescript.transaction(async t => {
+async function addServerInBlocklist (options: {
+  byAccountId: number
+  targetServerId: number
+
+  removeNotificationOfUserId: number | null
+}) {
+  const { byAccountId, targetServerId, removeNotificationOfUserId } = options
+
+  await sequelizeTypescript.transaction(async t => {
     return ServerBlocklistModel.upsert({
       accountId: byAccountId,
       targetServerId
     }, { transaction: t })
   })
+
+  UserNotificationModel.removeNotificationsOf({
+    id: targetServerId,
+    type: 'server',
+    forUserId: removeNotificationOfUserId
+  }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
 }
 
 function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) {

server/core/lib/emailer.ts

@@ -1,5 +1,5 @@
 import { arrayify } from '@peertube/peertube-core-utils'
-import { EmailPayload, SendEmailDefaultOptions, UserRegistrationState } from '@peertube/peertube-models'
+import { EmailPayload, SendEmailDefaultOptions, UserExportState, UserRegistrationState } from '@peertube/peertube-models'
 import { isTestOrDevInstance, root } from '@peertube/peertube-node-utils'
 import { readFileSync } from 'fs'
 import merge from 'lodash-es/merge.js'
@@ -8,8 +8,9 @@ import { join } from 'path'
 import { bunyanLogger, logger } from '../helpers/logger.js'
 import { CONFIG, isEmailEnabled } from '../initializers/config.js'
 import { WEBSERVER } from '../initializers/constants.js'
-import { MRegistration, MUser } from '../types/models/index.js'
+import { MRegistration, MUser, MUserExport, MUserImport } from '../types/models/index.js'
 import { JobQueue } from './job-queue/index.js'
+import { UserModel } from '@server/models/user/user.js'
 
 class Emailer {
 
@@ -52,6 +53,8 @@ class Emailer {
     }
   }
 
+  // ---------------------------------------------------------------------------
+
   addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) {
     const emailPayload: EmailPayload = {
       template: 'password-reset',
@@ -160,13 +163,82 @@ class Emailer {
       locals: {
         username: registration.username,
         moderationResponse: registration.moderationResponse,
-        loginLink: WEBSERVER.URL + '/login'
+        loginLink: WEBSERVER.URL + '/login',
+
+        hideNotificationPreferencesLink: true
       }
     }
 
     return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
   }
 
+  // ---------------------------------------------------------------------------
+
+  async addUserExportCompletedOrErroredJob (userExport: MUserExport) {
+    let template: string
+    let subject: string
+
+    if (userExport.state === UserExportState.COMPLETED) {
+      template = 'user-export-completed'
+      subject = `Your export archive has been created`
+    } else {
+      template = 'user-export-errored'
+      subject = `Failed to create your export archive`
+    }
+
+    const user = await UserModel.loadById(userExport.userId)
+
+    const emailPayload: EmailPayload = {
+      to: [ user.email ],
+      template,
+      subject,
+      locals: {
+        exportsUrl: WEBSERVER.URL + '/my-account/import-export',
+        errorMessage: userExport.error,
+
+        hideNotificationPreferencesLink: true
+      }
+    }
+
+    return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
+  }
+
+  async addUserImportErroredJob (userImport: MUserImport) {
+    const user = await UserModel.loadById(userImport.userId)
+
+    const emailPayload: EmailPayload = {
+      to: [ user.email ],
+      template: 'user-import-errored',
+      subject: 'Failed to import your archive',
+      locals: {
+        errorMessage: userImport.error,
+
+        hideNotificationPreferencesLink: true
+      }
+    }
+
+    return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
+  }
+
+  async addUserImportSuccessJob (userImport: MUserImport) {
+    const user = await UserModel.loadById(userImport.userId)
+
+    const emailPayload: EmailPayload = {
+      to: [ user.email ],
+      template: 'user-import-completed',
+      subject: 'Your archive import has finished',
+      locals: {
+        resultStats: userImport.resultSummary.stats,
+
+        hideNotificationPreferencesLink: true
+      }
+    }
+
+    return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
+  }
+
+  // ---------------------------------------------------------------------------
+
   async sendMail (options: EmailPayload) {
     if (!isEmailEnabled()) {
       logger.info('Cannot send mail because SMTP is not configured.')
@@ -233,14 +305,14 @@ class Emailer {
   private initSMTPTransport () {
     logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT)
 
-    let tls
+    let tls: { ca: [ Buffer ] }
     if (CONFIG.SMTP.CA_FILE) {
       tls = {
         ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ]
       }
     }
 
-    let auth
+    let auth: { user: string, pass: string }
     if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) {
       auth = {
         user: CONFIG.SMTP.USERNAME,

server/core/lib/emails/user-export-completed/html.pug

@@ -0,0 +1,9 @@
+extends ../common/greetings
+include ../common/mixins.pug
+
+block title
+  | Your export archive has been created
+
+block content
+  p
+    | Your export archive has been created. You can download it in #[a(href=exportsUrl) your account export page].

server/core/lib/emails/user-export-errored/html.pug

@@ -0,0 +1,12 @@
+extends ../common/greetings
+include ../common/mixins.pug
+
+block title
+  | Failed to create your export archive
+
+block content
+  p
+    | We are sorry but the generation of your export archive has failed:
+  blockquote !{errorMessage}
+  p
+    | Please contact your administrator if the problem occurs again.

server/core/lib/emails/user-import-completed/html.pug

@@ -0,0 +1,46 @@
+extends ../common/greetings
+include ../common/mixins.pug
+
+mixin displaySummary(stats)
+  ul
+    if stats.success
+      li Imported: #{stats.success}
+    if stats.duplicates
+      li Not imported as considered duplicate: #{stats.duplicates}
+    if stats.errors
+      li Not imported due to error: #{stats.errors}
+
+block title
+  | Your archive import has finished
+
+block content
+  p Your archive import has finished. Here is the summary of imported objects:
+
+  ul
+    li
+      strong User settings:
+      +displaySummary(resultStats.userSettings)
+    li
+      strong Account (name, description, avatar...):
+      +displaySummary(resultStats.account)
+    li
+      strong Blocklist:
+      +displaySummary(resultStats.blocklist)
+    li
+      strong Channels:
+      +displaySummary(resultStats.channels)
+    li
+      strong Likes:
+      +displaySummary(resultStats.likes)
+    li
+      strong Dislikes:
+      +displaySummary(resultStats.dislikes)
+    li
+      strong Subscriptions:
+      +displaySummary(resultStats.following)
+    li
+      strong Video Playlists:
+      +displaySummary(resultStats.videoPlaylists)
+    li
+      strong Videos:
+      +displaySummary(resultStats.videos)

server/core/lib/emails/user-import-errored/html.pug

@@ -0,0 +1,12 @@
+extends ../common/greetings
+include ../common/mixins.pug
+
+block title
+  | Failed to import your archive
+
+block content
+  p
+    | We are sorry but the import of your archive has failed:
+  blockquote !{errorMessage}
+  p
+    | Please contact your administrator if the problem occurs again.

server/core/lib/files-cache/video-captions-simple-file-cache.ts

@@ -1,7 +1,6 @@
 import { join } from 'path'
 import { logger } from '@server/helpers/logger.js'
 import { doRequestAndSaveToFile } from '@server/helpers/requests.js'
-import { CONFIG } from '../../initializers/config.js'
 import { FILES_CACHE } from '../../initializers/constants.js'
 import { VideoModel } from '../../models/video/video.js'
 import { VideoCaptionModel } from '../../models/video/video-caption.js'
@@ -24,7 +23,7 @@ class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> {
     if (!videoCaption) return undefined
 
     if (videoCaption.isOwned()) {
-      return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
+      return { isOwned: true, path: videoCaption.getFSPath() }
     }
 
     return this.loadRemoteFile(filename)

server/core/lib/hls.ts

@@ -1,6 +1,6 @@
 import { uniqify, uuidRegex } from '@peertube/peertube-core-utils'
 import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
-import { VideoStorage } from '@peertube/peertube-models'
+import { FileStorage } from '@peertube/peertube-models'
 import { sha256 } from '@peertube/peertube-node-utils'
 import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models/index.js'
 import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm'
@@ -100,7 +100,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
 
     logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid))
 
-    if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+    if (playlist.storage === FileStorage.OBJECT_STORAGE) {
       playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
       await remove(masterPlaylistPath)
     }
@@ -151,7 +151,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist
     const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
     await outputJSON(outputPath, json)
 
-    if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+    if (playlist.storage === FileStorage.OBJECT_STORAGE) {
       playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
       await remove(outputPath)
     }

server/core/lib/job-queue/handlers/activitypub-follow.ts

@@ -18,6 +18,10 @@ async function processActivityPubFollow (job: Job) {
   const payload = job.data as ActivitypubFollowPayload
   const host = payload.host
 
+  const handle = host
+    ? `${payload.name}@${host}`
+    : payload.name
+
   logger.info('Processing ActivityPub follow in job %s.', job.id)
 
   let targetActor: MActorFull
@@ -30,14 +34,24 @@ async function processActivityPubFollow (job: Job) {
 
     let actorUrl: string
 
+    try {
       if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
       if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
 
       targetActor = await getOrCreateAPActor(actorUrl, 'all')
+    } catch (err) {
+      logger.warn(`Do not follow ${handle} because we could not find the actor URL (in database or using webfinger)`)
+      return
+    }
+  }
+
+  if (!targetActor) {
+    logger.warn(`Do not follow ${handle} because we could not fetch/load the actor`)
+    return
   }
 
   if (payload.assertIsChannel && !targetActor.VideoChannel) {
-    logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host)
+    logger.warn(`Do not follow ${handle} because it is not a channel.`)
     return
   }
 

server/core/lib/job-queue/handlers/create-user-export.ts

@@ -0,0 +1,34 @@
+import { Job } from 'bullmq'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { CreateUserExportPayload } from '@peertube/peertube-models'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { UserExporter } from '@server/lib/user-import-export/user-exporter.js'
+import { Emailer } from '@server/lib/emailer.js'
+
+const lTags = loggerTagsFactory('user-export')
+
+export async function processCreateUserExport (job: Job): Promise<void> {
+  const payload = job.data as CreateUserExportPayload
+  const exportModel = await UserExportModel.load(payload.userExportId)
+
+  logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags())
+
+  if (!exportModel) {
+    logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags())
+    return
+  }
+
+  const exporter = new UserExporter()
+
+  try {
+    await exporter.export(exportModel)
+
+    await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
+
+    logger.info(`User export ${payload.userExportId} has been created`, lTags())
+  } catch (err) {
+    await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
+
+    throw err
+  }
+}

server/core/lib/job-queue/handlers/import-user-archive.ts

@@ -0,0 +1,33 @@
+import { Job } from 'bullmq'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { ImportUserArchivePayload } from '@peertube/peertube-models'
+import { UserImportModel } from '@server/models/user/user-import.js'
+import { UserImporter } from '@server/lib/user-import-export/user-importer.js'
+import { Emailer } from '@server/lib/emailer.js'
+
+const lTags = loggerTagsFactory('user-import')
+
+export async function processImportUserArchive (job: Job): Promise<void> {
+  const payload = job.data as ImportUserArchivePayload
+  const importModel = await UserImportModel.load(payload.userImportId)
+
+  logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags())
+
+  if (!importModel) {
+    logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags())
+    return
+  }
+
+  const exporter = new UserImporter()
+  await exporter.import(importModel)
+
+  try {
+    await Emailer.Instance.addUserImportSuccessJob(importModel)
+
+    logger.info(`User import ${payload.userImportId} ended`, lTags())
+  } catch (err) {
+    await Emailer.Instance.addUserImportErroredJob(importModel)
+
+    throw err
+  }
+}

server/core/lib/job-queue/handlers/move-to-file-system.ts

@@ -1,6 +1,6 @@
 import { Job } from 'bullmq'
 import { join } from 'path'
-import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
+import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
@@ -52,7 +52,7 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
 
 async function moveWebVideoFiles (video: MVideoWithAllFiles) {
   for (const file of video.VideoFiles) {
-    if (file.storage === VideoStorage.FILE_SYSTEM) continue
+    if (file.storage === FileStorage.FILE_SYSTEM) continue
 
     await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
     await onFileMoved({
@@ -68,7 +68,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
     const playlistWithVideo = playlist.withVideo(video)
 
     for (const file of playlist.VideoFiles) {
-      if (file.storage === VideoStorage.FILE_SYSTEM) continue
+      if (file.storage === FileStorage.FILE_SYSTEM) continue
 
       // Resolution playlist
       const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
@@ -97,7 +97,7 @@ async function onFileMoved (options: {
   const oldFileUrl = file.fileUrl
 
   file.fileUrl = null
-  file.storage = VideoStorage.FILE_SYSTEM
+  file.storage = FileStorage.FILE_SYSTEM
 
   await updateTorrentMetadata(videoOrPlaylist, file)
   await file.save()
@@ -114,7 +114,7 @@ async function doAfterLastMove (options: {
   const { video, previousVideoState, isNewVideo } = options
 
   for (const playlist of video.VideoStreamingPlaylists) {
-    if (playlist.storage === VideoStorage.FILE_SYSTEM) continue
+    if (playlist.storage === FileStorage.FILE_SYSTEM) continue
 
     const playlistWithVideo = playlist.withVideo(video)
 
@@ -124,7 +124,7 @@ async function doAfterLastMove (options: {
 
     playlist.playlistUrl = null
     playlist.segmentsSha256Url = null
-    playlist.storage = VideoStorage.FILE_SYSTEM
+    playlist.storage = FileStorage.FILE_SYSTEM
 
     playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
     playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION

server/core/lib/job-queue/handlers/move-to-object-storage.ts

@@ -1,7 +1,7 @@
 import { Job } from 'bullmq'
 import { remove } from 'fs-extra/esm'
 import { join } from 'path'
-import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
+import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
@@ -45,7 +45,7 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
 
 async function moveWebVideoFiles (video: MVideoWithAllFiles) {
   for (const file of video.VideoFiles) {
-    if (file.storage !== VideoStorage.FILE_SYSTEM) continue
+    if (file.storage !== FileStorage.FILE_SYSTEM) continue
 
     const fileUrl = await storeWebVideoFile(video, file)
 
@@ -59,7 +59,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
     const playlistWithVideo = playlist.withVideo(video)
 
     for (const file of playlist.VideoFiles) {
-      if (file.storage !== VideoStorage.FILE_SYSTEM) continue
+      if (file.storage !== FileStorage.FILE_SYSTEM) continue
 
       // Resolution playlist
       const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
@@ -84,7 +84,7 @@ async function onFileMoved (options: {
   const { videoOrPlaylist, file, fileUrl, oldPath } = options
 
   file.fileUrl = fileUrl
-  file.storage = VideoStorage.OBJECT_STORAGE
+  file.storage = FileStorage.OBJECT_STORAGE
 
   await updateTorrentMetadata(videoOrPlaylist, file)
   await file.save()
@@ -101,13 +101,13 @@ async function doAfterLastMove (options: {
   const { video, previousVideoState, isNewVideo } = options
 
   for (const playlist of video.VideoStreamingPlaylists) {
-    if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
+    if (playlist.storage === FileStorage.OBJECT_STORAGE) continue
 
     const playlistWithVideo = playlist.withVideo(video)
 
     playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
     playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
-    playlist.storage = VideoStorage.OBJECT_STORAGE
+    playlist.storage = FileStorage.OBJECT_STORAGE
 
     playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
     playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION

server/core/lib/job-queue/handlers/video-file-import.ts

@@ -1,13 +1,12 @@
 import { Job } from 'bullmq'
 import { copy } from 'fs-extra/esm'
 import { stat } from 'fs/promises'
-import { VideoFileImportPayload, VideoStorage } from '@peertube/peertube-models'
+import { VideoFileImportPayload, FileStorage } from '@peertube/peertube-models'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
 import { generateWebVideoFilename } from '@server/lib/paths.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { buildMoveJob } from '@server/lib/video.js'
 import { VideoFileModel } from '@server/models/video/video-file.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MVideoFullLight } from '@server/types/models/index.js'
@@ -15,6 +14,7 @@ import { getLowercaseExtension } from '@peertube/peertube-node-utils'
 import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
 import { logger } from '../../../helpers/logger.js'
 import { JobQueue } from '../job-queue.js'
+import { buildMoveJob } from '@server/lib/video-jobs.js'
 
 async function processVideoFileImport (job: Job) {
   const payload = job.data as VideoFileImportPayload
@@ -68,7 +68,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
     resolution,
     extname: fileExt,
     filename: generateWebVideoFilename(resolution, fileExt),
-    storage: VideoStorage.FILE_SYSTEM,
+    storage: FileStorage.FILE_SYSTEM,
     size,
     fps,
     videoId: video.id

server/core/lib/job-queue/handlers/video-import.ts

@@ -22,10 +22,10 @@ import { generateWebVideoFilename } from '@server/lib/paths.js'
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { ServerConfigManager } from '@server/lib/server-config-manager.js'
 import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
-import { isAbleToUploadVideo } from '@server/lib/user.js'
+import { isUserQuotaValid } from '@server/lib/user.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
-import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js'
+import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
 import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
 import { getLowercaseExtension } from '@peertube/peertube-node-utils'
@@ -138,7 +138,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
     // Get information about this video
     const stats = await stat(tempVideoPath)
-    const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size)
+    const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size })
     if (isAble === false) {
       throw new Error('The user video quota is exceeded with this video to import.')
     }

server/core/lib/job-queue/handlers/video-live-ending.ts

@@ -30,7 +30,7 @@ import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoS
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
 import { JobQueue } from '../job-queue.js'
 import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
-import { buildStoryboardJobIfNeeded } from '@server/lib/video.js'
+import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 
 const lTags = loggerTagsFactory('live', 'job')
 

server/core/lib/job-queue/handlers/video-studio-edition.ts

@@ -4,7 +4,7 @@ import { join } from 'path'
 import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
-import { isAbleToUploadVideo } from '@server/lib/user.js'
+import { isUserQuotaValid } from '@server/lib/user.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
 import { UserModel } from '@server/models/user/user.js'
@@ -170,7 +170,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud
   const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file
 
   const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder)
-  if (await isAbleToUploadVideo(user.id, additionalBytes) === false) {
+  if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) {
     throw new Error('Quota exceeded for this user to edit the video')
   }
 }

server/core/lib/job-queue/job-queue.ts

@@ -18,10 +18,12 @@ import {
   ActivitypubHttpUnicastPayload,
   ActorKeysPayload,
   AfterVideoChannelImportPayload,
+  CreateUserExportPayload,
   DeleteResumableUploadMetaFilePayload,
   EmailPayload,
   FederateVideoPayload,
   GenerateStoryboardPayload,
+  ImportUserArchivePayload,
   JobState,
   JobType,
   ManageVideoTorrentPayload,
@@ -71,6 +73,8 @@ import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
 import { processVideoTranscoding } from './handlers/video-transcoding.js'
 import { processVideosViewsStats } from './handlers/video-views-stats.js'
 import { onMoveToFileSystemFailure, processMoveToFileSystem } from './handlers/move-to-file-system.js'
+import { processCreateUserExport } from './handlers/create-user-export.js'
+import { processImportUserArchive } from './handlers/import-user-archive.js'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -98,7 +102,9 @@ export type CreateJobArgument =
   { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
   { type: 'federate-video', payload: FederateVideoPayload } |
-  { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
+  { type: 'create-user-export', payload: CreateUserExportPayload } |
+  { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } |
+  { type: 'import-user-archive', payload: ImportUserArchivePayload }
 
 export type CreateJobOptions = {
   delay?: number
@@ -131,7 +137,9 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'video-studio-edition': processVideoStudioEdition,
   'video-transcoding': processVideoTranscoding,
   'videos-views-stats': processVideosViewsStats,
-  'generate-video-storyboard': processGenerateStoryboard
+  'generate-video-storyboard': processGenerateStoryboard,
+  'create-user-export': processCreateUserExport,
+  'import-user-archive': processImportUserArchive
 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@@ -164,7 +172,9 @@ const jobTypes: JobType[] = [
   'video-redundancy',
   'video-studio-edition',
   'video-transcoding',
-  'videos-views-stats'
+  'videos-views-stats',
+  'create-user-export',
+  'import-user-archive'
 ]
 
 const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])

server/core/lib/live/live-utils.ts

@@ -1,7 +1,7 @@
 import { pathExists, remove } from 'fs-extra/esm'
 import { readdir } from 'fs/promises'
 import { basename, join } from 'path'
-import { LiveVideoLatencyMode, LiveVideoLatencyModeType, VideoStorage } from '@peertube/peertube-models'
+import { LiveVideoLatencyMode, LiveVideoLatencyModeType, FileStorage } from '@peertube/peertube-models'
 import { logger } from '@server/helpers/logger.js'
 import { VIDEO_LIVE } from '@server/initializers/constants.js'
 import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'
@@ -24,7 +24,7 @@ async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStre
   const hlsDirectory = getLiveDirectory(video)
 
   // We uploaded files to object storage too, remove them
-  if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+  if (streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
     await removeHLSObjectStorage(streamingPlaylist.withVideo(video))
   }
 
@@ -86,7 +86,7 @@ async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
 }
 
 async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
-  if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
+  if (streamingPlaylist.storage !== FileStorage.OBJECT_STORAGE) return
 
   logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid)
 

server/core/lib/live/shared/muxing-session.ts

@@ -14,14 +14,14 @@ import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFile
 import { VideoFileModel } from '@server/models/video/video-file.js'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models/index.js'
-import { LiveVideoError, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
+import { LiveVideoError, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
 import {
   generateHLSMasterPlaylistFilename,
   generateHlsSha256SegmentsFilename,
   getLiveDirectory,
   getLiveReplayBaseDirectory
 } from '../../paths.js'
-import { isAbleToUploadVideo } from '../../user.js'
+import { isUserQuotaValid } from '../../user.js'
 import { LiveQuotaStore } from '../live-quota-store.js'
 import { LiveSegmentShaStore } from '../live-segment-sha-store.js'
 import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils.js'
@@ -95,7 +95,7 @@ class MuxingSession extends EventEmitter {
   private aborted = false
 
   private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
-    return isAbleToUploadVideo(userId, 1000)
+    return isUserQuotaValid({ userId, uploadSize: 1000 })
   }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
 
   private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => {
@@ -186,7 +186,7 @@ class MuxingSession extends EventEmitter {
       if (this.masterPlaylistCreated === true) return
 
       try {
-        if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+        if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
           let masterContent = await readFile(path, 'utf-8')
 
           // If the disk sync is slow, don't upload an empty master playlist on object storage
@@ -260,7 +260,7 @@ class MuxingSession extends EventEmitter {
         logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() })
       }
 
-      if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+      if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
         try {
           await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath)
         } catch (err) {
@@ -345,7 +345,7 @@ class MuxingSession extends EventEmitter {
       await this.addSegmentToReplay(segmentPath)
     }
 
-    if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+    if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
       try {
         await storeHLSFileFromPath(this.streamingPlaylist, segmentPath)
 
@@ -464,8 +464,8 @@ class MuxingSession extends EventEmitter {
     playlist.type = VideoStreamingPlaylistType.HLS
 
     playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED
-      ? VideoStorage.OBJECT_STORAGE
-      : VideoStorage.FILE_SYSTEM
+      ? FileStorage.OBJECT_STORAGE
+      : FileStorage.FILE_SYSTEM
 
     return playlist.save()
   }

server/core/lib/local-actor.ts

@@ -30,13 +30,16 @@ export function buildActorInstance (type: ActivityPubActorType, url: string, pre
   }) as MActor
 }
 
-export async function updateLocalActorImageFiles (
-  accountOrChannel: MAccountDefault | MChannelDefault,
-  imagePhysicalFile: Express.Multer.File,
+export async function updateLocalActorImageFiles (options: {
+  accountOrChannel: MAccountDefault | MChannelDefault
+  imagePhysicalFile: { path: string }
   type: ActorImageType_Type
-) {
+  sendActorUpdate: boolean
+}) {
+  const { accountOrChannel, imagePhysicalFile, type, sendActorUpdate } = options
+
   const processImageSize = async (imageSize: { width: number, height: number }) => {
-    const extension = getLowercaseExtension(imagePhysicalFile.filename)
+    const extension = getLowercaseExtension(imagePhysicalFile.path)
 
     const imageName = buildUUID() + extension
     const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName)
@@ -63,7 +66,9 @@ export async function updateLocalActorImageFiles (
     const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t)
     await updatedActor.save({ transaction: t })
 
+    if (sendActorUpdate) {
       await sendUpdateActor(accountOrChannel, t)
+    }
 
     return type === ActorImageType.AVATAR
       ? updatedActor.Avatars

server/core/lib/model-loaders/video.ts

@@ -1,3 +1,4 @@
+import { CONFIG } from '@server/initializers/config.js'
 import { VideoModel } from '@server/models/video/video.js'
 import {
   MVideoAccountLightBlacklistAllFiles,
@@ -7,6 +8,7 @@ import {
   MVideoImmutable,
   MVideoThumbnail
 } from '@server/types/models/index.js'
+import { getOrCreateAPVideo } from '../activitypub/videos/get.js'
 
 type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes'
 
@@ -50,17 +52,36 @@ function loadVideoByUrl (
   url: string,
   fetchType: VideoLoadByUrlType
 ): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
-  if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
+  if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccountAndFiles(url)
 
   if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
 
   if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
 }
 
+async function loadOrCreateVideoIfAllowedForUser (videoUrl: string) {
+  if (CONFIG.SEARCH.REMOTE_URI.USERS) {
+    try {
+      const res = await getOrCreateAPVideo({
+        videoObject: videoUrl,
+        fetchType: 'only-immutable-attributes',
+        allowRefresh: false
+      })
+
+      return res?.video
+    } catch {
+      return undefined
+    }
+  }
+
+  return VideoModel.loadByUrlImmutableAttributes(videoUrl)
+}
+
 export {
   type VideoLoadType,
   type VideoLoadByUrlType,
 
   loadVideo,
-  loadVideoByUrl
+  loadVideoByUrl,
+  loadOrCreateVideoIfAllowedForUser
 }

server/core/lib/moderation.ts

@@ -17,6 +17,7 @@ import {
   MCommentAbuseAccountVideo,
   MCommentOwnerVideo,
   MUser,
+  MUserDefault,
   MVideoAbuseVideoFull,
   MVideoAccountLightBlacklistAllFiles
 } from '@server/types/models/index.js'
@@ -38,7 +39,7 @@ export type AcceptResult = {
 function isLocalVideoFileAccepted (object: {
   videoBody: VideoCreate
   videoFile: VideoUploadFile
-  user: UserModel
+  user: MUserDefault
 }): AcceptResult {
   return { accepted: true }
 }

server/core/lib/object-storage/keys.ts

@@ -13,8 +13,13 @@ function generateWebVideoObjectStorageKey (filename: string) {
   return filename
 }
 
+function generateUserExportObjectStorageKey (filename: string) {
+  return filename
+}
+
 export {
   generateHLSObjectStorageKey,
   generateHLSObjectBaseStorageKey,
-  generateWebVideoObjectStorageKey
+  generateWebVideoObjectStorageKey,
+  generateUserExportObjectStorageKey
 }