From 8573e5a80a8a24628ede7e6e43ecd389c2b5b3b4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 12 Feb 2024 10:47:52 +0100 Subject: [PATCH] Implement user import/export in server --- config/default.yaml | 32 +- config/production.yaml.example | 30 +- package.json | 9 +- packages/core-utils/src/common/url.ts | 13 + .../src/activitypub/activitypub-actor.ts | 6 +- .../src/activitypub/activitypub-collection.ts | 2 +- .../activitypub-ordered-collection.ts | 4 +- .../src/activitypub/objects/video-object.ts | 10 + .../models/src/common/file-storage.enum.ts | 6 + packages/models/src/common/index.ts | 1 + packages/models/src/import-export/index.ts | 9 + .../account-export.model.ts | 18 + .../actor-export.model.ts | 6 + .../blocklist-export.model.ts | 9 + .../channel-export.model.ts | 23 ++ .../comments-export.model.ts | 12 + .../dislikes-export.model.ts | 8 + .../followers-export.model.ts | 9 + .../following-export.model.ts | 9 + .../peertube-export-format/index.ts | 12 + .../likes-export.model.ts | 8 + .../user-settings-export.model.ts | 26 ++ .../video-export.model.ts | 103 ++++++ .../video-playlists-export.model.ts | 34 ++ .../user-export-request-result.model.ts | 5 + .../user-export-request.model.ts | 3 + .../import-export/user-export-state.enum.ts | 8 + .../src/import-export/user-export.model.ts | 18 + .../import-export/user-import-result.model.ts | 20 ++ .../import-export/user-import-state.enum.ts | 8 + .../user-import-upload-result.model.ts | 5 + .../src/import-export/user-import.model.ts | 10 + packages/models/src/index.ts | 1 + .../src/plugins/server/server-hook.model.ts | 4 + .../models/src/server/custom-config.model.ts | 14 +- packages/models/src/server/debug.model.ts | 1 + packages/models/src/server/job.model.ts | 14 + .../models/src/server/server-config.model.ts | 13 + .../src/server/server-error-code.enum.ts | 4 +- packages/models/src/users/user-right.enum.ts | 5 +- packages/models/src/videos/index.ts | 1 - .../models/src/videos/video-storage.enum.ts | 6 - packages/node-utils/src/path.ts | 14 +- packages/typescript-utils/src/types.ts | 2 + scripts/i18n/create-custom-files.ts | 4 + server/core/controllers/api/config.ts | 10 + server/core/controllers/api/index.ts | 1 - .../controllers/api/runners/jobs-files.ts | 4 +- .../controllers/api/search/search-videos.ts | 2 +- server/core/controllers/api/server/debug.ts | 2 + .../api/server/server-blocklist.ts | 30 +- server/core/controllers/api/users/index.ts | 4 + server/core/controllers/api/users/me.ts | 11 +- .../controllers/api/users/my-blocklist.ts | 28 +- .../controllers/api/users/my-notifications.ts | 16 +- .../controllers/api/users/user-exports.ts | 100 ++++++ .../controllers/api/users/user-imports.ts | 90 +++++ server/core/controllers/api/video-channel.ts | 15 +- server/core/controllers/api/video-playlist.ts | 1 - server/core/controllers/api/videos/rate.ts | 71 +--- server/core/controllers/api/videos/source.ts | 2 +- server/core/controllers/api/videos/update.ts | 3 +- server/core/controllers/api/videos/upload.ts | 60 +--- server/core/controllers/download.ts | 70 +++- server/core/helpers/captions-utils.ts | 9 +- server/core/helpers/unzip.ts | 55 +++ server/core/initializers/config.ts | 15 + server/core/initializers/constants.ts | 47 ++- server/core/initializers/database.ts | 6 +- .../migrations/0660-object-storage.ts | 6 +- .../migrations/0810-user-export.ts | 33 ++ .../migrations/0815-user-import.ts | 31 ++ server/core/lib/activitypub/collection.ts | 13 +- .../lib/activitypub/process/process-delete.ts | 2 +- .../lib/activitypub/process/process-flag.ts | 2 +- server/core/lib/activitypub/videos/refresh.ts | 2 +- server/core/lib/blocklist.ts | 36 +- server/core/lib/emailer.ts | 82 ++++- .../lib/emails/user-export-completed/html.pug | 9 + .../lib/emails/user-export-errored/html.pug | 12 + .../lib/emails/user-import-completed/html.pug | 46 +++ .../lib/emails/user-import-errored/html.pug | 12 + .../video-captions-simple-file-cache.ts | 3 +- server/core/lib/hls.ts | 6 +- .../job-queue/handlers/activitypub-follow.ts | 22 +- .../job-queue/handlers/create-user-export.ts | 34 ++ .../job-queue/handlers/import-user-archive.ts | 33 ++ .../job-queue/handlers/move-to-file-system.ts | 12 +- .../handlers/move-to-object-storage.ts | 12 +- .../job-queue/handlers/video-file-import.ts | 6 +- .../lib/job-queue/handlers/video-import.ts | 6 +- .../job-queue/handlers/video-live-ending.ts | 2 +- .../handlers/video-studio-edition.ts | 4 +- server/core/lib/job-queue/job-queue.ts | 16 +- server/core/lib/live/live-utils.ts | 6 +- server/core/lib/live/shared/muxing-session.ts | 16 +- server/core/lib/local-actor.ts | 17 +- server/core/lib/model-loaders/video.ts | 25 +- server/core/lib/moderation.ts | 3 +- server/core/lib/object-storage/keys.ts | 7 +- .../lib/object-storage/pre-signed-urls.ts | 62 +++- .../shared/object-storage-helpers.ts | 43 ++- server/core/lib/object-storage/user-export.ts | 25 ++ server/core/lib/paths.ts | 53 ++- .../lib/plugins/plugin-helpers-builder.ts | 21 +- server/core/lib/rate.ts | 64 ++++ .../remove-expired-user-exports-scheduler.ts | 30 ++ .../lib/schedulers/update-videos-scheduler.ts | 2 +- server/core/lib/server-config-manager.ts | 10 + server/core/lib/thumbnail.ts | 2 +- .../core/lib/transcoding/web-transcoding.ts | 6 +- .../exporters/abstract-user-exporter.ts | 51 +++ .../exporters/account-exporter.ts | 68 ++++ .../exporters/actor-exporter.ts | 78 +++++ .../exporters/blocklist-exporter.ts | 24 ++ .../exporters/channels-exporter.ts | 64 ++++ .../exporters/comments-exporter.ts | 50 +++ .../exporters/dislikes-exporter.ts | 41 +++ .../exporters/followers-exporter.ts | 45 +++ .../exporters/following-exporter.ts | 49 +++ .../lib/user-import-export/exporters/index.ts | 12 + .../exporters/likes-exporter.ts | 40 +++ .../exporters/user-settings-exporter.ts | 33 ++ .../exporters/video-playlists-exporter.ts | 75 ++++ .../exporters/videos-exporter.ts | 329 ++++++++++++++++++ .../importers/abstract-rates-importer.ts | 30 ++ .../importers/abstract-user-importer.ts | 119 +++++++ .../importers/account-blocklist-importer.ts | 66 ++++ .../importers/account-importer.ts | 54 +++ .../importers/channels-importer.ts | 74 ++++ .../importers/dislikes-importer.ts | 17 + .../importers/following-importer.ts | 37 ++ .../lib/user-import-export/importers/index.ts | 9 + .../importers/likes-importer.ts | 17 + .../importers/user-settings-importer.ts | 88 +++++ .../importers/video-playlists-importer.ts | 169 +++++++++ .../importers/videos-importer.ts | 314 +++++++++++++++++ .../lib/user-import-export/user-exporter.ts | 282 +++++++++++++++ .../lib/user-import-export/user-importer.ts | 145 ++++++++ server/core/lib/user.ts | 41 +-- server/core/lib/video-jobs.ts | 164 +++++++++ server/core/lib/video-path-manager.ts | 8 +- server/core/lib/video-privacy.ts | 6 +- server/core/lib/video-state.ts | 2 +- server/core/lib/video-studio.ts | 2 +- server/core/lib/video.ts | 120 +------ server/core/middlewares/validators/config.ts | 5 + server/core/middlewares/validators/index.ts | 9 +- .../middlewares/validators/shared/videos.ts | 4 +- .../middlewares/validators/users/index.ts | 7 + .../validators/users/shared/index.ts | 1 + .../{ => users}/shared/user-registrations.ts | 0 .../{ => users}/user-email-verification.ts | 6 +- .../validators/users/user-exports.ts | 155 +++++++++ .../validators/{ => users}/user-history.ts | 4 +- .../validators/users/user-import.ts | 129 +++++++ .../{ => users}/user-notifications.ts | 6 +- .../{ => users}/user-registrations.ts | 10 +- .../{ => users}/user-subscriptions.ts | 8 +- .../validators/{ => users}/users.ts | 18 +- .../videos/shared/video-validators.ts | 3 +- .../validators/videos/video-rates.ts | 1 + .../core/models/account/account-blocklist.ts | 11 +- .../core/models/account/account-video-rate.ts | 127 ++++--- server/core/models/actor/actor-follow.ts | 136 ++++++-- server/core/models/actor/actor.ts | 15 + server/core/models/user/user-export.ts | 213 ++++++++++++ server/core/models/user/user-import.ts | 88 +++++ .../models/user/user-notification-setting.ts | 10 + server/core/models/user/user.ts | 55 ++- .../formatter/video-activity-pub-format.ts | 35 +- .../video/video-model-get-query-builder.ts | 3 +- server/core/models/video/thumbnail.ts | 20 +- server/core/models/video/video-caption.ts | 20 +- server/core/models/video/video-channel.ts | 4 +- server/core/models/video/video-comment.ts | 26 ++ server/core/models/video/video-file.ts | 26 +- server/core/models/video/video-live.ts | 18 +- .../models/video/video-playlist-element.ts | 34 +- server/core/models/video/video-playlist.ts | 44 ++- .../models/video/video-streaming-playlist.ts | 16 +- server/core/models/video/video.ts | 27 +- server/core/types/express.d.ts | 8 +- server/core/types/models/user/index.ts | 2 + server/core/types/models/user/user-export.ts | 5 + server/core/types/models/user/user-import.ts | 5 + .../core/types/models/video/video-caption.ts | 3 +- .../core/types/models/video/video-comment.ts | 7 + server/core/types/models/video/video-live.ts | 4 + .../models/video/video-playlist-element.ts | 6 +- server/core/types/models/video/video-rate.ts | 6 +- .../scripts/create-generate-storyboard-job.ts | 2 +- .../scripts/create-move-video-storage-job.ts | 6 +- server/server.ts | 2 + tsconfig.base.json | 1 - yarn.lock | 264 +++++++++++++- 196 files changed, 5661 insertions(+), 722 deletions(-) create mode 100644 packages/models/src/common/file-storage.enum.ts create mode 100644 packages/models/src/import-export/index.ts create mode 100644 packages/models/src/import-export/peertube-export-format/account-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/actor-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/blocklist-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/channel-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/comments-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/dislikes-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/followers-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/following-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/index.ts create mode 100644 packages/models/src/import-export/peertube-export-format/likes-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/user-settings-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/video-export.model.ts create mode 100644 packages/models/src/import-export/peertube-export-format/video-playlists-export.model.ts create mode 100644 packages/models/src/import-export/user-export-request-result.model.ts create mode 100644 packages/models/src/import-export/user-export-request.model.ts create mode 100644 packages/models/src/import-export/user-export-state.enum.ts create mode 100644 packages/models/src/import-export/user-export.model.ts create mode 100644 packages/models/src/import-export/user-import-result.model.ts create mode 100644 packages/models/src/import-export/user-import-state.enum.ts create mode 100644 packages/models/src/import-export/user-import-upload-result.model.ts create mode 100644 packages/models/src/import-export/user-import.model.ts delete mode 100644 packages/models/src/videos/video-storage.enum.ts create mode 100644 server/core/controllers/api/users/user-exports.ts create mode 100644 server/core/controllers/api/users/user-imports.ts create mode 100644 server/core/helpers/unzip.ts create mode 100644 server/core/initializers/migrations/0810-user-export.ts create mode 100644 server/core/initializers/migrations/0815-user-import.ts create mode 100644 server/core/lib/emails/user-export-completed/html.pug create mode 100644 server/core/lib/emails/user-export-errored/html.pug create mode 100644 server/core/lib/emails/user-import-completed/html.pug create mode 100644 server/core/lib/emails/user-import-errored/html.pug create mode 100644 server/core/lib/job-queue/handlers/create-user-export.ts create mode 100644 server/core/lib/job-queue/handlers/import-user-archive.ts create mode 100644 server/core/lib/object-storage/user-export.ts create mode 100644 server/core/lib/rate.ts create mode 100644 server/core/lib/schedulers/remove-expired-user-exports-scheduler.ts create mode 100644 server/core/lib/user-import-export/exporters/abstract-user-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/account-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/actor-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/blocklist-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/channels-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/comments-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/dislikes-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/followers-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/following-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/index.ts create mode 100644 server/core/lib/user-import-export/exporters/likes-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/user-settings-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/video-playlists-exporter.ts create mode 100644 server/core/lib/user-import-export/exporters/videos-exporter.ts create mode 100644 server/core/lib/user-import-export/importers/abstract-rates-importer.ts create mode 100644 server/core/lib/user-import-export/importers/abstract-user-importer.ts create mode 100644 server/core/lib/user-import-export/importers/account-blocklist-importer.ts create mode 100644 server/core/lib/user-import-export/importers/account-importer.ts create mode 100644 server/core/lib/user-import-export/importers/channels-importer.ts create mode 100644 server/core/lib/user-import-export/importers/dislikes-importer.ts create mode 100644 server/core/lib/user-import-export/importers/following-importer.ts create mode 100644 server/core/lib/user-import-export/importers/index.ts create mode 100644 server/core/lib/user-import-export/importers/likes-importer.ts create mode 100644 server/core/lib/user-import-export/importers/user-settings-importer.ts create mode 100644 server/core/lib/user-import-export/importers/video-playlists-importer.ts create mode 100644 server/core/lib/user-import-export/importers/videos-importer.ts create mode 100644 server/core/lib/user-import-export/user-exporter.ts create mode 100644 server/core/lib/user-import-export/user-importer.ts create mode 100644 server/core/lib/video-jobs.ts create mode 100644 server/core/middlewares/validators/users/index.ts create mode 100644 server/core/middlewares/validators/users/shared/index.ts rename server/core/middlewares/validators/{ => users}/shared/user-registrations.ts (100%) rename server/core/middlewares/validators/{ => users}/user-email-verification.ts (96%) create mode 100644 server/core/middlewares/validators/users/user-exports.ts rename server/core/middlewares/validators/{ => users}/user-history.ts (87%) create mode 100644 server/core/middlewares/validators/users/user-import.ts rename server/core/middlewares/validators/{ => users}/user-notifications.ts (88%) rename server/core/middlewares/validators/{ => users}/user-registrations.ts (96%) rename server/core/middlewares/validators/{ => users}/user-subscriptions.ts (92%) rename server/core/middlewares/validators/{ => users}/users.ts (96%) create mode 100644 server/core/models/user/user-export.ts create mode 100644 server/core/models/user/user-import.ts create mode 100644 server/core/types/models/user/user-export.ts create mode 100644 server/core/types/models/user/user-import.ts diff --git a/config/default.yaml b/config/default.yaml index 5aeb25d5e..4ec113158 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -1,4 +1,4 @@ -# /!\ YOU SHOULD NOT UPDATE THIS FILE, USE production.yaml instead /!\ # +# /!\ DO NOT UPDATE THIS FILE, USE production.yaml instead /!\ # listen: hostname: '127.0.0.1' @@ -222,12 +222,16 @@ object_storage: # Useful when you want to use a CDN/external proxy base_url: '' # Example: 'https://mirror.example.com' - # Same settings but for web videos web_videos: bucket_name: 'web-videos' prefix: '' base_url: '' + user_exports: + bucket_name: 'user-exports' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' @@ -482,11 +486,14 @@ user: videos: # Enable or disable video history by default for new users. enabled: true - # Default value of maximum video bytes the user can upload (does not take into account transcoded files) + + # Default value of maximum video bytes the user can upload + # Does not take into account transcoded files or account export archives (that can include user uploaded files) # Byte format is supported ("1GB" etc) # -1 == unlimited video_quota: -1 video_quota_daily: -1 + default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username video_channels: @@ -707,6 +714,24 @@ import: # Max number of videos to import when the user asks for full sync full_sync_videos_limit: 1000 + users: + # Video quota is checked on import so the user doesn't upload a too big archive file + # Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import + enabled: true + +export: + users: + # Allow users to export their PeerTube data in a .zip for backup or re-import + # Only one export at a time is allowed per user + enabled: true + + # Max size of the current user quota to accept or not the export + # Goal of this setting is to not store too big archive file on your server disk + max_user_video_quota: 10GB + + # How long PeerTube should keep the user export + export_expiration: '2 days' + auto_blacklist: # New videos automatically blacklisted so moderators can review before publishing videos: @@ -867,6 +892,7 @@ client: # By default PeerTube client displays author username prefer_author_display_name: false display_author_avatar: false + resumable_upload: # Max size of upload chunks, e.g. '90MB' # If null, it will be calculated based on network speed diff --git a/config/production.yaml.example b/config/production.yaml.example index dca6f7308..28f4cf7a3 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -220,12 +220,16 @@ object_storage: # Useful when you want to use a CDN/external proxy base_url: '' # Example: 'https://mirror.example.com' - # Same settings but for web videos web_videos: bucket_name: 'web-videos' prefix: '' base_url: '' + user_exports: + bucket_name: 'user-exports' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' @@ -492,11 +496,14 @@ user: videos: # Enable or disable video history by default for new users. enabled: true - # Default value of maximum video bytes the user can upload (does not take into account transcoded files) + + # Default value of maximum video bytes the user can upload + # Does not take into account transcoded files or account export archives (that can include user uploaded files) # Byte format is supported ("1GB" etc) # -1 == unlimited video_quota: -1 video_quota_daily: -1 + default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username video_channels: @@ -717,6 +724,24 @@ import: # Max number of videos to import when the user asks for full sync full_sync_videos_limit: 1000 + users: + # Video quota is checked on import so the user doesn't upload a too big archive file + # Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import + enabled: true + +export: + users: + # Allow users to export their PeerTube data in a .zip for backup or re-import + # Only one export at a time is allowed per user + enabled: true + + # Max size of the current user quota to accept or not the export + # Goal of this setting is to not store too big archive file on your server disk + max_user_video_quota: 10GB + + # How long PeerTube should keep the user export + export_expiration: '2 days' + auto_blacklist: # New videos automatically blacklisted so moderators can review before publishing videos: @@ -877,6 +902,7 @@ client: # By default PeerTube client displays author username prefer_author_display_name: false display_author_avatar: false + resumable_upload: # Max size of upload chunks, e.g. '90MB' # If null, it will be calculated based on network speed diff --git a/package.json b/package.json index decfc6529..7119dfbf0 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "@peertube/http-signature": "^1.7.0", "@smithy/node-http-handler": "^2.1.7", "@uploadx/core": "^6.0.0", + "archiver": "^6.0.1", "async-mutex": "^0.4.0", "bcrypt": "5.1.1", "bencode": "^4.0.0", @@ -142,6 +143,7 @@ "jimp": "^0.22.4", "js-yaml": "^4.0.0", "jsonld": "~8.3.1", + "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", "lru-cache": "^10.0.1", "magnet-uri": "^7.0.5", @@ -178,11 +180,13 @@ "webfinger.js": "^2.6.6", "webtorrent": "^2.1.27", "winston": "3.11.0", - "ws": "^8.0.0" + "ws": "^8.0.0", + "yauzl": "^2.10.0" }, "devDependencies": { "@peertube/maildev": "^1.2.0", "@peertube/resolve-tspaths": "^0.8.14", + "@types/archiver": "^6.0.2", "@types/bcrypt": "^5.0.0", "@types/bencode": "^2.0.0", "@types/bluebird": "^3.5.33", @@ -197,6 +201,7 @@ "@types/fluent-ffmpeg": "^2.1.16", "@types/fs-extra": "^11.0.1", "@types/jsonld": "^1.5.9", + "@types/jsonwebtoken": "^9.0.5", "@types/lodash-es": "^4.17.8", "@types/magnet-uri": "^5.1.1", "@types/maildev": "^0.0.4", @@ -212,6 +217,7 @@ "@types/validator": "^13.9.0", "@types/webtorrent": "^0.109.0", "@types/ws": "^8.2.0", + "@types/yauzl": "^2.10.3", "@typescript-eslint/eslint-plugin": "^6.7.5", "autocannon": "^7.0.4", "chai": "^4.1.1", @@ -228,6 +234,7 @@ "eslint-plugin-promise": "^6.0.0", "fast-xml-parser": "^4.0.0-beta.8", "jpeg-js": "^0.4.4", + "jszip": "^3.10.1", "mocha": "^10.0.0", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", diff --git a/packages/core-utils/src/common/url.ts b/packages/core-utils/src/common/url.ts index 449b6c9dc..9b1b0650f 100644 --- a/packages/core-utils/src/common/url.ts +++ b/packages/core-utils/src/common/url.ts @@ -19,6 +19,18 @@ function removeQueryParams (url: string) { return objUrl.toString() } +function queryParamsToObject (entries: any) { + const result: { [ id: string ]: string | number | boolean } = {} + + for (const [ key, value ] of entries) { + result[key] = value + } + + return result +} + +// --------------------------------------------------------------------------- + function buildPlaylistLink (playlist: Pick, base?: string) { return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist) } @@ -123,6 +135,7 @@ function decoratePlaylistLink (options: { export { addQueryParams, removeQueryParams, + queryParamsToObject, buildPlaylistLink, buildVideoLink, diff --git a/packages/models/src/activitypub/activitypub-actor.ts b/packages/models/src/activitypub/activitypub-actor.ts index 85b37c5ad..6340de287 100644 --- a/packages/models/src/activitypub/activitypub-actor.ts +++ b/packages/models/src/activitypub/activitypub-actor.ts @@ -18,7 +18,7 @@ export interface ActivityPubActor { sharedInbox: string } summary: string - attributedTo: ActivityPubAttributedTo[] + attributedTo?: ActivityPubAttributedTo[] support?: string publicKey: { @@ -31,4 +31,8 @@ export interface ActivityPubActor { icon?: ActivityIconObject | ActivityIconObject[] published?: string + + // For export + likes?: string + dislikes?: string } diff --git a/packages/models/src/activitypub/activitypub-collection.ts b/packages/models/src/activitypub/activitypub-collection.ts index b98ad37c2..640b3f09d 100644 --- a/packages/models/src/activitypub/activitypub-collection.ts +++ b/packages/models/src/activitypub/activitypub-collection.ts @@ -1,7 +1,7 @@ import { Activity } from './activity.js' export interface ActivityPubCollection { - '@context': string[] + '@context': any[] type: 'Collection' | 'CollectionPage' totalItems: number partOf?: string diff --git a/packages/models/src/activitypub/activitypub-ordered-collection.ts b/packages/models/src/activitypub/activitypub-ordered-collection.ts index 3de0890bb..b2aa6c1db 100644 --- a/packages/models/src/activitypub/activitypub-ordered-collection.ts +++ b/packages/models/src/activitypub/activitypub-ordered-collection.ts @@ -1,5 +1,7 @@ export interface ActivityPubOrderedCollection { - '@context': string[] + id: string + + '@context': any[] type: 'OrderedCollection' | 'OrderedCollectionPage' totalItems: number orderedItems: T[] diff --git a/packages/models/src/activitypub/objects/video-object.ts b/packages/models/src/activitypub/objects/video-object.ts index 754f69f83..fd852fbed 100644 --- a/packages/models/src/activitypub/objects/video-object.ts +++ b/packages/models/src/activitypub/objects/video-object.ts @@ -59,6 +59,16 @@ export interface VideoObject { to?: string[] cc?: string[] + + // For export + attachment?: { + type: 'Video' + url: string + mediaType: string + height: number + size: number + fps: number + }[] } export interface ActivityPubStoryboard { diff --git a/packages/models/src/common/file-storage.enum.ts b/packages/models/src/common/file-storage.enum.ts new file mode 100644 index 000000000..bf0eff21d --- /dev/null +++ b/packages/models/src/common/file-storage.enum.ts @@ -0,0 +1,6 @@ +export const FileStorage = { + FILE_SYSTEM: 0, + OBJECT_STORAGE: 1 +} as const + +export type FileStorageType = typeof FileStorage[keyof typeof FileStorage] diff --git a/packages/models/src/common/index.ts b/packages/models/src/common/index.ts index 957851ae4..1f8ebdd94 100644 --- a/packages/models/src/common/index.ts +++ b/packages/models/src/common/index.ts @@ -1 +1,2 @@ +export * from './file-storage.enum.js' export * from './result-list.model.js' diff --git a/packages/models/src/import-export/index.ts b/packages/models/src/import-export/index.ts new file mode 100644 index 000000000..8f38321af --- /dev/null +++ b/packages/models/src/import-export/index.ts @@ -0,0 +1,9 @@ +export * from './peertube-export-format/index.js' +export * from './user-export-request-result.model.js' +export * from './user-export-request.model.js' +export * from './user-export-state.enum.js' +export * from './user-export.model.js' +export * from './user-import.model.js' +export * from './user-import-state.enum.js' +export * from './user-import-result.model.js' +export * from './user-import-upload-result.model.js' diff --git a/packages/models/src/import-export/peertube-export-format/account-export.model.ts b/packages/models/src/import-export/peertube-export-format/account-export.model.ts new file mode 100644 index 000000000..3b06a3c70 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/account-export.model.ts @@ -0,0 +1,18 @@ +import { UserActorImageJSON } from './actor-export.model.js' + +export interface AccountExportJSON { + url: string + + name: string + displayName: string + description: string + + updatedAt: string + createdAt: string + + avatars: UserActorImageJSON[] + + archiveFiles: { + avatar: string | null + } +} diff --git a/packages/models/src/import-export/peertube-export-format/actor-export.model.ts b/packages/models/src/import-export/peertube-export-format/actor-export.model.ts new file mode 100644 index 000000000..e88be7132 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/actor-export.model.ts @@ -0,0 +1,6 @@ +export interface UserActorImageJSON { + width: number + url: string + createdAt: string + updatedAt: string +} diff --git a/packages/models/src/import-export/peertube-export-format/blocklist-export.model.ts b/packages/models/src/import-export/peertube-export-format/blocklist-export.model.ts new file mode 100644 index 000000000..fc3a0808e --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/blocklist-export.model.ts @@ -0,0 +1,9 @@ +export interface BlocklistExportJSON { + instances: { + host: string + }[] + + actors: { + handle: string + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/channel-export.model.ts b/packages/models/src/import-export/peertube-export-format/channel-export.model.ts new file mode 100644 index 000000000..99aa36f75 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/channel-export.model.ts @@ -0,0 +1,23 @@ +import { UserActorImageJSON } from './actor-export.model.js' + +export interface ChannelExportJSON { + channels: { + url: string + + name: string + displayName: string + description: string + support: string + + updatedAt: string + createdAt: string + + avatars: UserActorImageJSON[] + banners: UserActorImageJSON[] + + archiveFiles: { + avatar: string | null + banner: string | null + } + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/comments-export.model.ts b/packages/models/src/import-export/peertube-export-format/comments-export.model.ts new file mode 100644 index 000000000..02ace4e3b --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/comments-export.model.ts @@ -0,0 +1,12 @@ +export interface CommentsExportJSON { + comments: { + url: string + text: string + createdAt: string + videoUrl: string + + inReplyToCommentUrl?: string + + archiveFiles?: never + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/dislikes-export.model.ts b/packages/models/src/import-export/peertube-export-format/dislikes-export.model.ts new file mode 100644 index 000000000..a732d8492 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/dislikes-export.model.ts @@ -0,0 +1,8 @@ +export interface DislikesExportJSON { + dislikes: { + videoUrl: string + createdAt: string + + archiveFiles?: never + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/followers-export.model.ts b/packages/models/src/import-export/peertube-export-format/followers-export.model.ts new file mode 100644 index 000000000..326b17670 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/followers-export.model.ts @@ -0,0 +1,9 @@ +export interface FollowersExportJSON { + followers: { + handle: string + createdAt: string + targetHandle: string + + archiveFiles?: never + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/following-export.model.ts b/packages/models/src/import-export/peertube-export-format/following-export.model.ts new file mode 100644 index 000000000..46287e0e8 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/following-export.model.ts @@ -0,0 +1,9 @@ +export interface FollowingExportJSON { + following: { + handle: string + targetHandle: string + createdAt: string + + archiveFiles?: never + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/index.ts b/packages/models/src/import-export/peertube-export-format/index.ts new file mode 100644 index 000000000..099520651 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/index.ts @@ -0,0 +1,12 @@ +export * from './account-export.model.js' +export * from './actor-export.model.js' +export * from './blocklist-export.model.js' +export * from './channel-export.model.js' +export * from './comments-export.model.js' +export * from './dislikes-export.model.js' +export * from './followers-export.model.js' +export * from './following-export.model.js' +export * from './likes-export.model.js' +export * from './user-settings-export.model.js' +export * from './video-export.model.js' +export * from './video-playlists-export.model.js' diff --git a/packages/models/src/import-export/peertube-export-format/likes-export.model.ts b/packages/models/src/import-export/peertube-export-format/likes-export.model.ts new file mode 100644 index 000000000..8f78247c0 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/likes-export.model.ts @@ -0,0 +1,8 @@ +export interface LikesExportJSON { + likes: { + videoUrl: string + createdAt: string + + archiveFiles?: never + }[] +} diff --git a/packages/models/src/import-export/peertube-export-format/user-settings-export.model.ts b/packages/models/src/import-export/peertube-export-format/user-settings-export.model.ts new file mode 100644 index 000000000..0eb3b6e98 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/user-settings-export.model.ts @@ -0,0 +1,26 @@ +import { UserNotificationSetting } from '../../users/user-notification-setting.model.js' +import { NSFWPolicyType } from '../../videos/nsfw-policy.type.js' + +export interface UserSettingsExportJSON { + email: string + + emailPublic: boolean + nsfwPolicy: NSFWPolicyType + + autoPlayVideo: boolean + autoPlayNextVideo: boolean + autoPlayNextVideoPlaylist: boolean + + p2pEnabled: boolean + + videosHistoryEnabled: boolean + videoLanguages: string[] + + theme: string + + createdAt: Date + + notificationSettings: UserNotificationSetting + + archiveFiles?: never +} diff --git a/packages/models/src/import-export/peertube-export-format/video-export.model.ts b/packages/models/src/import-export/peertube-export-format/video-export.model.ts new file mode 100644 index 000000000..8f0d18797 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/video-export.model.ts @@ -0,0 +1,103 @@ +import { + LiveVideoLatencyModeType, + VideoPrivacyType, + VideoStateType, + VideoStreamingPlaylistType_Type +} from '../../videos/index.js' + +export interface VideoExportJSON { + videos: { + uuid: string + + createdAt: string + updatedAt: string + publishedAt: string + originallyPublishedAt: string + + name: string + category: number + licence: number + language: string + tags: string[] + + privacy: VideoPrivacyType + passwords: string[] + + duration: number + + description: string + support: string + + isLive: boolean + live?: { + saveReplay: boolean + permanentLive: boolean + latencyMode: LiveVideoLatencyModeType + streamKey: string + + replaySettings?: { + privacy: VideoPrivacyType + } + } + + url: string + + thumbnailUrl: string + previewUrl: string + + views: number + + likes: number + dislikes: number + + nsfw: boolean + + commentsEnabled: boolean + downloadEnabled: boolean + + channel: { + name: string + } + + waitTranscoding: boolean + state: VideoStateType + + captions: { + createdAt: string + updatedAt: string + language: string + filename: string + fileUrl: string + }[] + + files: VideoFileExportJSON[] + + streamingPlaylists: { + type: VideoStreamingPlaylistType_Type + playlistUrl: string + segmentsSha256Url: string + files: VideoFileExportJSON[] + }[] + + source?: { + filename: string + } + + archiveFiles: { + videoFile: string | null + thumbnail: string | null + captions: Record // The key is the language code + } + }[] +} + +// --------------------------------------------------------------------------- + +export interface VideoFileExportJSON { + resolution: number + size: number // Bytes + fps: number + + torrentUrl: string + fileUrl: string +} diff --git a/packages/models/src/import-export/peertube-export-format/video-playlists-export.model.ts b/packages/models/src/import-export/peertube-export-format/video-playlists-export.model.ts new file mode 100644 index 000000000..6ee27b395 --- /dev/null +++ b/packages/models/src/import-export/peertube-export-format/video-playlists-export.model.ts @@ -0,0 +1,34 @@ +import { VideoPlaylistPrivacyType } from '../../videos/playlist/video-playlist-privacy.model.js' +import { VideoPlaylistType_Type } from '../../videos/playlist/video-playlist-type.model.js' + +export interface VideoPlaylistsExportJSON { + videoPlaylists: { + displayName: string + description: string + privacy: VideoPlaylistPrivacyType + url: string + uuid: string + + type: VideoPlaylistType_Type + + channel: { + name: string + } + + createdAt: string + updatedAt: string + + thumbnailUrl: string + + elements: { + videoUrl: string + + startTimestamp?: number + stopTimestamp?: number + }[] + + archiveFiles: { + thumbnail: string | null + } + }[] +} diff --git a/packages/models/src/import-export/user-export-request-result.model.ts b/packages/models/src/import-export/user-export-request-result.model.ts new file mode 100644 index 000000000..fac4ae816 --- /dev/null +++ b/packages/models/src/import-export/user-export-request-result.model.ts @@ -0,0 +1,5 @@ +export interface UserExportRequestResult { + export: { + id: number + } +} diff --git a/packages/models/src/import-export/user-export-request.model.ts b/packages/models/src/import-export/user-export-request.model.ts new file mode 100644 index 000000000..85190ee7b --- /dev/null +++ b/packages/models/src/import-export/user-export-request.model.ts @@ -0,0 +1,3 @@ +export interface UserExportRequest { + withVideoFiles: boolean +} diff --git a/packages/models/src/import-export/user-export-state.enum.ts b/packages/models/src/import-export/user-export-state.enum.ts new file mode 100644 index 000000000..c74bde726 --- /dev/null +++ b/packages/models/src/import-export/user-export-state.enum.ts @@ -0,0 +1,8 @@ +export const UserExportState = { + PENDING: 1, + PROCESSING: 2, + COMPLETED: 3, + ERRORED: 4 +} as const + +export type UserExportStateType = typeof UserExportState[keyof typeof UserExportState] diff --git a/packages/models/src/import-export/user-export.model.ts b/packages/models/src/import-export/user-export.model.ts new file mode 100644 index 000000000..40c18d3d3 --- /dev/null +++ b/packages/models/src/import-export/user-export.model.ts @@ -0,0 +1,18 @@ +import { UserExportStateType } from './user-export-state.enum.js' + +export interface UserExport { + id: number + + state: { + id: UserExportStateType + label: string + } + + // In bytes + size: number + + privateDownloadUrl: string + + createdAt: string | Date + expiresOn: string | Date +} diff --git a/packages/models/src/import-export/user-import-result.model.ts b/packages/models/src/import-export/user-import-result.model.ts new file mode 100644 index 000000000..57e387f55 --- /dev/null +++ b/packages/models/src/import-export/user-import-result.model.ts @@ -0,0 +1,20 @@ +type Summary = { + success: number + duplicates: number + errors: number +} + +export interface UserImportResultSummary { + stats: { + blocklist: Summary + channels: Summary + likes: Summary + dislikes: Summary + following: Summary + videoPlaylists: Summary + videos: Summary + + account: Summary + userSettings: Summary + } +} diff --git a/packages/models/src/import-export/user-import-state.enum.ts b/packages/models/src/import-export/user-import-state.enum.ts new file mode 100644 index 000000000..454c0e69d --- /dev/null +++ b/packages/models/src/import-export/user-import-state.enum.ts @@ -0,0 +1,8 @@ +export const UserImportState = { + PENDING: 1, + PROCESSING: 2, + COMPLETED: 3, + ERRORED: 4 +} as const + +export type UserImportStateType = typeof UserImportState[keyof typeof UserImportState] diff --git a/packages/models/src/import-export/user-import-upload-result.model.ts b/packages/models/src/import-export/user-import-upload-result.model.ts new file mode 100644 index 000000000..d9c1e53d0 --- /dev/null +++ b/packages/models/src/import-export/user-import-upload-result.model.ts @@ -0,0 +1,5 @@ +export interface UserImportUploadResult { + userImport: { + id: number + } +} diff --git a/packages/models/src/import-export/user-import.model.ts b/packages/models/src/import-export/user-import.model.ts new file mode 100644 index 000000000..134758027 --- /dev/null +++ b/packages/models/src/import-export/user-import.model.ts @@ -0,0 +1,10 @@ +import { UserImportStateType } from './user-import-state.enum.js' + +export interface UserImport { + id: number + state: { + id: UserImportStateType + label: string + } + createdAt: string +} diff --git a/packages/models/src/index.ts b/packages/models/src/index.ts index b76703dff..8a1ec8dcb 100644 --- a/packages/models/src/index.ts +++ b/packages/models/src/index.ts @@ -3,6 +3,7 @@ export * from './actors/index.js' export * from './bulk/index.js' export * from './common/index.js' export * from './custom-markup/index.js' +export * from './import-export/index.js' export * from './feeds/index.js' export * from './http/index.js' export * from './joinpeertube/index.js' diff --git a/packages/models/src/plugins/server/server-hook.model.ts b/packages/models/src/plugins/server/server-hook.model.ts index cf387ffd7..4b71b9531 100644 --- a/packages/models/src/plugins/server/server-hook.model.ts +++ b/packages/models/src/plugins/server/server-hook.model.ts @@ -65,6 +65,8 @@ export const serverFilterHookObject = { 'filter:api.video.post-import-url.accept.result': true, 'filter:api.video.post-import-torrent.accept.result': true, 'filter:api.video.update-file.accept.result': true, + // PeerTube >= 6.1 + 'filter:api.video.user-import.accept.result': true, // Filter the result of the accept comment (thread or reply) functions // If the functions return false then the user cannot post its comment 'filter:api.video-thread.create.accept.result': true, @@ -75,6 +77,8 @@ export const serverFilterHookObject = { 'filter:api.video.import-url.video-attribute.result': true, 'filter:api.video.import-torrent.video-attribute.result': true, 'filter:api.video.live.video-attribute.result': true, + // PeerTube >= 6.1 + 'filter:api.video.user-import.video-attribute.result': true, // Filter params/result used to list threads of a specific video // (used by the video watch page) diff --git a/packages/models/src/server/custom-config.model.ts b/packages/models/src/server/custom-config.model.ts index 0fce36d5c..3f52fb31a 100644 --- a/packages/models/src/server/custom-config.model.ts +++ b/packages/models/src/server/custom-config.model.ts @@ -193,10 +193,23 @@ export interface CustomConfig { enabled: boolean } } + videoChannelSynchronization: { enabled: boolean maxPerUser: number } + + users: { + enabled: boolean + } + } + + export: { + users: { + enabled: boolean + maxUserVideoQuota: number + exportExpiration: number + } } trending: { @@ -260,5 +273,4 @@ export interface CustomConfig { storyboards: { enabled: boolean } - } diff --git a/packages/models/src/server/debug.model.ts b/packages/models/src/server/debug.model.ts index 41f2109af..067e53c41 100644 --- a/packages/models/src/server/debug.model.ts +++ b/packages/models/src/server/debug.model.ts @@ -9,4 +9,5 @@ export interface SendDebugCommand { | 'process-video-viewers' | 'process-video-channel-sync-latest' | 'process-update-videos-scheduler' + | 'remove-expired-user-exports' } diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts index 913686d72..7713a9d3e 100644 --- a/packages/models/src/server/job.model.ts +++ b/packages/models/src/server/job.model.ts @@ -31,6 +31,8 @@ export type JobType = | 'video-transcoding' | 'videos-views-stats' | 'generate-video-storyboard' + | 'create-user-export' + | 'import-user-archive' export interface Job { id: number | string @@ -302,3 +304,15 @@ export interface GenerateStoryboardPayload { videoUUID: string federate: boolean } + +// --------------------------------------------------------------------------- + +export interface CreateUserExportPayload { + userExportId: number +} + +// --------------------------------------------------------------------------- + +export interface ImportUserArchivePayload { + userImportId: number +} diff --git a/packages/models/src/server/server-config.model.ts b/packages/models/src/server/server-config.model.ts index 6fff58647..db1fbd072 100644 --- a/packages/models/src/server/server-config.model.ts +++ b/packages/models/src/server/server-config.model.ts @@ -207,9 +207,22 @@ export interface ServerConfig { enabled: boolean } } + videoChannelSynchronization: { enabled: boolean } + + users: { + enabled:boolean + } + } + + export: { + users: { + enabled: boolean + exportExpiration: number + maxUserVideoQuota: number + } } autoBlacklist: { diff --git a/packages/models/src/server/server-error-code.enum.ts b/packages/models/src/server/server-error-code.enum.ts index dc200c1ea..156525c79 100644 --- a/packages/models/src/server/server-error-code.enum.ts +++ b/packages/models/src/server/server-error-code.enum.ts @@ -54,7 +54,9 @@ export const ServerErrorCode = { VIDEO_REQUIRES_PASSWORD:'video_requires_password', INCORRECT_VIDEO_PASSWORD:'incorrect_video_password', - VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded' + VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded', + + MAX_USER_VIDEO_QUOTA_EXCEEDED_FOR_USER_EXPORT: 'max_user_video_quota_exceeded_for_user_export' } as const /** diff --git a/packages/models/src/users/user-right.enum.ts b/packages/models/src/users/user-right.enum.ts index 534b9feb0..1da8b64f9 100644 --- a/packages/models/src/users/user-right.enum.ts +++ b/packages/models/src/users/user-right.enum.ts @@ -47,7 +47,10 @@ export const UserRight = { MANAGE_REGISTRATIONS: 28, - MANAGE_RUNNERS: 29 + MANAGE_RUNNERS: 29, + + MANAGE_USER_EXPORTS: 30, + MANAGE_USER_IMPORTS: 31 } as const export type UserRightType = typeof UserRight[keyof typeof UserRight] diff --git a/packages/models/src/videos/index.ts b/packages/models/src/videos/index.ts index 7d96d31a6..24f4cf5c7 100644 --- a/packages/models/src/videos/index.ts +++ b/packages/models/src/videos/index.ts @@ -29,7 +29,6 @@ export * from './video-rate.type.js' export * from './video-schedule-update.model.js' export * from './video-sort-field.type.js' export * from './video-state.enum.js' -export * from './video-storage.enum.js' export * from './video-source.model.js' export * from './video-streaming-playlist.model.js' diff --git a/packages/models/src/videos/video-storage.enum.ts b/packages/models/src/videos/video-storage.enum.ts deleted file mode 100644 index de5c92e0d..000000000 --- a/packages/models/src/videos/video-storage.enum.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const VideoStorage = { - FILE_SYSTEM: 0, - OBJECT_STORAGE: 1 -} as const - -export type VideoStorageType = typeof VideoStorage[keyof typeof VideoStorage] diff --git a/packages/node-utils/src/path.ts b/packages/node-utils/src/path.ts index 1d569833e..ae84666dd 100644 --- a/packages/node-utils/src/path.ts +++ b/packages/node-utils/src/path.ts @@ -1,4 +1,4 @@ -import { basename, extname, isAbsolute, join, resolve } from 'path' +import { basename, extname, isAbsolute, join, parse, resolve } from 'path' import { fileURLToPath } from 'url' let rootPath: string @@ -48,3 +48,15 @@ export function buildAbsoluteFixturePath (path: string, customCIPath = false) { return join(root(), 'packages', 'tests', 'fixtures', path) } + +export function getFilenameFromUrl (url: string) { + return getFilename(new URL(url).pathname) +} + +export function getFilename (path: string) { + return parse(path).base +} + +export function getFilenameWithoutExt (path: string) { + return parse(path).name +} diff --git a/packages/typescript-utils/src/types.ts b/packages/typescript-utils/src/types.ts index cd998a467..0b9694ad4 100644 --- a/packages/typescript-utils/src/types.ts +++ b/packages/typescript-utils/src/types.ts @@ -45,3 +45,5 @@ export type DeepOmitArray = { } export type Unpacked = T extends (infer U)[] ? U : T + +export type Awaitable = T | PromiseLike diff --git a/scripts/i18n/create-custom-files.ts b/scripts/i18n/create-custom-files.ts index 68cde7dd3..a692f034f 100755 --- a/scripts/i18n/create-custom-files.ts +++ b/scripts/i18n/create-custom-files.ts @@ -6,6 +6,7 @@ import { ABUSE_STATES, buildLanguages, RUNNER_JOB_STATES, + USER_EXPORT_STATES, USER_REGISTRATION_STATES, VIDEO_CATEGORIES, VIDEO_CHANNEL_SYNC_STATE, @@ -14,6 +15,7 @@ import { VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES, VIDEO_PRIVACIES, + USER_IMPORT_STATES, VIDEO_STATES } from '@peertube/peertube-server/core/initializers/constants.js' @@ -96,6 +98,8 @@ Object.values(VIDEO_CATEGORIES) .concat(Object.values(ABUSE_STATES)) .concat(Object.values(USER_REGISTRATION_STATES)) .concat(Object.values(RUNNER_JOB_STATES)) + .concat(Object.values(USER_EXPORT_STATES)) + .concat(Object.values(USER_IMPORT_STATES)) .concat([ 'This video does not exist.', 'We cannot fetch the video. Please try again later.', diff --git a/server/core/controllers/api/config.ts b/server/core/controllers/api/config.ts index 6630cb40b..eece4dbe3 100644 --- a/server/core/controllers/api/config.ts +++ b/server/core/controllers/api/config.ts @@ -355,6 +355,16 @@ function customConfig (): CustomConfig { videoChannelSynchronization: { enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED, maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER + }, + users: { + enabled: CONFIG.IMPORT.USERS.ENABLED + } + }, + export: { + users: { + enabled: CONFIG.EXPORT.USERS.ENABLED, + exportExpiration: CONFIG.EXPORT.USERS.EXPORT_EXPIRATION, + maxUserVideoQuota: CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA } }, trending: { diff --git a/server/core/controllers/api/index.ts b/server/core/controllers/api/index.ts index cb5d59802..b48a6645e 100644 --- a/server/core/controllers/api/index.ts +++ b/server/core/controllers/api/index.ts @@ -50,7 +50,6 @@ apiRouter.use('/custom-pages', customPageRouter) apiRouter.use('/blocklist', blocklistRouter) apiRouter.use('/runners', runnersRouter) -// apiRouter.use(apiRateLimiter) apiRouter.use('/ping', pong) apiRouter.use('/*', badRequest) diff --git a/server/core/controllers/api/runners/jobs-files.ts b/server/core/controllers/api/runners/jobs-files.ts index 91a7302b8..a27128d00 100644 --- a/server/core/controllers/api/runners/jobs-files.ts +++ b/server/core/controllers/api/runners/jobs-files.ts @@ -9,7 +9,7 @@ import { runnerJobGetVideoStudioTaskFileValidator, runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files.js' -import { RunnerJobState, VideoStorage } from '@peertube/peertube-models' +import { RunnerJobState, FileStorage } from '@peertube/peertube-models' const lTags = loggerTagsFactory('api', 'runner') @@ -57,7 +57,7 @@ async function getMaxQualityVideoFile (req: express.Request, res: express.Respon const file = video.getMaxQualityFile() - if (file.storage === VideoStorage.OBJECT_STORAGE) { + if (file.storage === FileStorage.OBJECT_STORAGE) { if (file.isHLS()) { return proxifyHLS({ req, diff --git a/server/core/controllers/api/search/search-videos.ts b/server/core/controllers/api/search/search-videos.ts index e3be237af..b8f521c1a 100644 --- a/server/core/controllers/api/search/search-videos.ts +++ b/server/core/controllers/api/search/search-videos.ts @@ -151,7 +151,7 @@ async function searchVideoURI (url: string, res: express.Response) { logger.info('Cannot search remote video %s.', url, { err }) } } else { - video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url)) + video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccountAndFiles(url)) } return res.json({ diff --git a/server/core/controllers/api/server/debug.ts b/server/core/controllers/api/server/debug.ts index 072dbab23..5ecf21d6b 100644 --- a/server/core/controllers/api/server/debug.ts +++ b/server/core/controllers/api/server/debug.ts @@ -7,6 +7,7 @@ import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-ch import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler.js' import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' import { authenticate, ensureUserHasRight } from '../../../middlewares/index.js' +import { RemoveExpiredUserExportsScheduler } from '@server/lib/schedulers/remove-expired-user-exports-scheduler.js' const debugRouter = express.Router() @@ -42,6 +43,7 @@ async function runCommand (req: express.Request, res: express.Response) { const processors: { [id in SendDebugCommand['command']]: () => Promise } = { 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), + 'remove-expired-user-exports': () => RemoveExpiredUserExportsScheduler.Instance.execute(), 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(), diff --git a/server/core/controllers/api/server/server-blocklist.ts b/server/core/controllers/api/server/server-blocklist.ts index 9ca7ec03e..a9ebd6627 100644 --- a/server/core/controllers/api/server/server-blocklist.ts +++ b/server/core/controllers/api/server/server-blocklist.ts @@ -1,9 +1,7 @@ import 'multer' import express from 'express' import { HttpStatusCode, UserRight } from '@peertube/peertube-models' -import { logger } from '@server/helpers/logger.js' import { getServerActor } from '@server/models/application/application.js' -import { UserNotificationModel } from '@server/models/user/user-notification.js' import { getFormattedObjects } from '../../../helpers/utils.js' import { addAccountInBlocklist, @@ -105,15 +103,9 @@ async function blockAccount (req: express.Request, res: express.Response) { const serverActor = await getServerActor() const accountToBlock = res.locals.account - await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id) + await addAccountInBlocklist({ byAccountId: serverActor.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: null }) - UserNotificationModel.removeNotificationsOf({ - id: accountToBlock.id, - type: 'account', - forUserId: null // For all users - }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function unblockAccount (req: express.Request, res: express.Response) { @@ -121,7 +113,7 @@ async function unblockAccount (req: express.Request, res: express.Response) { await removeAccountFromBlocklist(accountBlock) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function listBlockedServers (req: express.Request, res: express.Response) { @@ -142,15 +134,13 @@ async function blockServer (req: express.Request, res: express.Response) { const serverActor = await getServerActor() const serverToBlock = res.locals.server - await addServerInBlocklist(serverActor.Account.id, serverToBlock.id) + await addServerInBlocklist({ + byAccountId: serverActor.Account.id, + targetServerId: serverToBlock.id, + removeNotificationOfUserId: null + }) - UserNotificationModel.removeNotificationsOf({ - id: serverToBlock.id, - type: 'server', - forUserId: null // For all users - }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function unblockServer (req: express.Request, res: express.Response) { @@ -158,5 +148,5 @@ async function unblockServer (req: express.Request, res: express.Response) { await removeServerFromBlocklist(serverBlock) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } diff --git a/server/core/controllers/api/users/index.ts b/server/core/controllers/api/users/index.ts index e5290fe25..d44cbb7c9 100644 --- a/server/core/controllers/api/users/index.ts +++ b/server/core/controllers/api/users/index.ts @@ -47,6 +47,8 @@ import { mySubscriptionsRouter } from './my-subscriptions.js' import { myVideoPlaylistsRouter } from './my-video-playlists.js' import { registrationsRouter } from './registrations.js' import { twoFactorRouter } from './two-factor.js' +import { userExportsRouter } from './user-exports.js' +import { userImportRouter } from './user-imports.js' const auditLogger = auditLoggerFactory('users') @@ -55,6 +57,8 @@ const usersRouter = express.Router() usersRouter.use(apiRateLimiter) usersRouter.use('/', emailVerificationRouter) +usersRouter.use('/', userExportsRouter) +usersRouter.use('/', userImportRouter) usersRouter.use('/', registrationsRouter) usersRouter.use('/', twoFactorRouter) usersRouter.use('/', tokensRouter) diff --git a/server/core/controllers/api/users/me.ts b/server/core/controllers/api/users/me.ts index 69edd342d..461867876 100644 --- a/server/core/controllers/api/users/me.ts +++ b/server/core/controllers/api/users/me.ts @@ -262,11 +262,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response) { const userAccount = await AccountModel.load(user.Account.id) - const avatars = await updateLocalActorImageFiles( - userAccount, - avatarPhysicalFile, - ActorImageType.AVATAR - ) + const avatars = await updateLocalActorImageFiles({ + accountOrChannel: userAccount, + imagePhysicalFile: avatarPhysicalFile, + type: ActorImageType.AVATAR, + sendActorUpdate: true + }) return res.json({ avatars: avatars.map(avatar => avatar.toFormattedJSON()) diff --git a/server/core/controllers/api/users/my-blocklist.ts b/server/core/controllers/api/users/my-blocklist.ts index 46a988ea6..ba10fb29d 100644 --- a/server/core/controllers/api/users/my-blocklist.ts +++ b/server/core/controllers/api/users/my-blocklist.ts @@ -1,8 +1,6 @@ import 'multer' import express from 'express' import { HttpStatusCode } from '@peertube/peertube-models' -import { logger } from '@server/helpers/logger.js' -import { UserNotificationModel } from '@server/models/user/user-notification.js' import { getFormattedObjects } from '../../../helpers/utils.js' import { addAccountInBlocklist, @@ -97,15 +95,9 @@ async function blockAccount (req: express.Request, res: express.Response) { const user = res.locals.oauth.token.User const accountToBlock = res.locals.account - await addAccountInBlocklist(user.Account.id, accountToBlock.id) + await addAccountInBlocklist({ byAccountId: user.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: user.id }) - UserNotificationModel.removeNotificationsOf({ - id: accountToBlock.id, - type: 'account', - forUserId: user.id - }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function unblockAccount (req: express.Request, res: express.Response) { @@ -134,15 +126,13 @@ async function blockServer (req: express.Request, res: express.Response) { const user = res.locals.oauth.token.User const serverToBlock = res.locals.server - await addServerInBlocklist(user.Account.id, serverToBlock.id) + await addServerInBlocklist({ + byAccountId: user.Account.id, + targetServerId: serverToBlock.id, + removeNotificationOfUserId: user.id + }) - UserNotificationModel.removeNotificationsOf({ - id: serverToBlock.id, - type: 'server', - forUserId: user.id - }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function unblockServer (req: express.Request, res: express.Response) { @@ -150,5 +140,5 @@ async function unblockServer (req: express.Request, res: express.Response) { await removeServerFromBlocklist(serverBlock) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } diff --git a/server/core/controllers/api/users/my-notifications.ts b/server/core/controllers/api/users/my-notifications.ts index c0172a452..3266dd811 100644 --- a/server/core/controllers/api/users/my-notifications.ts +++ b/server/core/controllers/api/users/my-notifications.ts @@ -16,7 +16,7 @@ import { listUserNotificationsValidator, markAsReadUserNotificationsValidator, updateNotificationSettingsValidator -} from '../../../middlewares/validators/user-notifications.js' +} from '../../../middlewares/validators/users/user-notifications.js' import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting.js' import { meRouter } from './me.js' @@ -59,12 +59,6 @@ async function updateNotificationSettings (req: express.Request, res: express.Re const user = res.locals.oauth.token.User const body = req.body as UserNotificationSetting - const query = { - where: { - userId: user.id - } - } - const values: UserNotificationSetting = { newVideoFromSubscription: body.newVideoFromSubscription, newCommentOnMyVideo: body.newCommentOnMyVideo, @@ -85,9 +79,9 @@ async function updateNotificationSettings (req: express.Request, res: express.Re myVideoStudioEditionFinished: body.myVideoStudioEditionFinished } - await UserNotificationSettingModel.update(values, query) + await UserNotificationSettingModel.updateUserSettings(values, user.id) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function listUserNotifications (req: express.Request, res: express.Response) { @@ -103,7 +97,7 @@ async function markAsReadUserNotifications (req: express.Request, res: express.R await UserNotificationModel.markAsRead(user.id, req.body.ids) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) { @@ -111,5 +105,5 @@ async function markAsReadAllUserNotifications (req: express.Request, res: expres await UserNotificationModel.markAllAsRead(user.id) - return res.status(HttpStatusCode.NO_CONTENT_204).end() + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } diff --git a/server/core/controllers/api/users/user-exports.ts b/server/core/controllers/api/users/user-exports.ts new file mode 100644 index 000000000..52ea7b725 --- /dev/null +++ b/server/core/controllers/api/users/user-exports.ts @@ -0,0 +1,100 @@ +import express from 'express' +import { FileStorage, HttpStatusCode, UserExportRequest, UserExportRequestResult, UserExportState } from '@peertube/peertube-models' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + userExportDeleteValidator, + userExportRequestValidator, + userExportsListValidator +} from '../../../middlewares/index.js' +import { UserExportModel } from '@server/models/user/user-export.js' +import { getFormattedObjects } from '@server/helpers/utils.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { JobQueue } from '@server/lib/job-queue/job-queue.js' +import { CONFIG } from '@server/initializers/config.js' + +const userExportsRouter = express.Router() + +userExportsRouter.use(apiRateLimiter) + +userExportsRouter.post('/:userId/exports/request', + authenticate, + asyncMiddleware(userExportRequestValidator), + asyncMiddleware(requestExport) +) + +userExportsRouter.get('/:userId/exports', + authenticate, + asyncMiddleware(userExportsListValidator), + asyncMiddleware(listUserExports) +) + +userExportsRouter.delete('/:userId/exports/:id', + authenticate, + asyncMiddleware(userExportDeleteValidator), + asyncMiddleware(deleteUserExport) +) + +// --------------------------------------------------------------------------- + +export { + userExportsRouter +} + +// --------------------------------------------------------------------------- + +async function requestExport (req: express.Request, res: express.Response) { + const body = req.body as UserExportRequest + + const exportModel = new UserExportModel({ + state: UserExportState.PENDING, + withVideoFiles: body.withVideoFiles, + + storage: CONFIG.OBJECT_STORAGE.ENABLED + ? FileStorage.OBJECT_STORAGE + : FileStorage.FILE_SYSTEM, + + userId: res.locals.user.id, + createdAt: new Date() + }) + exportModel.generateAndSetFilename() + + await sequelizeTypescript.transaction(async transaction => { + await exportModel.save({ transaction }) + }) + + await JobQueue.Instance.createJob({ type: 'create-user-export', payload: { userExportId: exportModel.id } }) + + return res.json({ + export: { + id: exportModel.id + } + } as UserExportRequestResult) +} + +async function listUserExports (req: express.Request, res: express.Response) { + const resultList = await UserExportModel.listForApi({ + start: req.query.start, + count: req.query.count, + user: res.locals.user + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function deleteUserExport (req: express.Request, res: express.Response) { + const userExport = res.locals.userExport + + await sequelizeTypescript.transaction(async transaction => { + await userExport.reload({ transaction }) + + if (!userExport.canBeSafelyRemoved()) { + return res.sendStatus(HttpStatusCode.CONFLICT_409) + } + + await userExport.destroy({ transaction }) + }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/core/controllers/api/users/user-imports.ts b/server/core/controllers/api/users/user-imports.ts new file mode 100644 index 000000000..5bcac7e57 --- /dev/null +++ b/server/core/controllers/api/users/user-imports.ts @@ -0,0 +1,90 @@ +import express from 'express' +import { + apiRateLimiter, + asyncMiddleware, + authenticate +} from '../../../middlewares/index.js' +import { uploadx } from '@server/lib/uploadx.js' +import { + getLatestImportStatusValidator, + userImportRequestResumableInitValidator, + userImportRequestResumableValidator +} from '@server/middlewares/validators/users/user-import.js' +import { HttpStatusCode, UserImportState, UserImportUploadResult } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { UserImportModel } from '@server/models/user/user-import.js' +import { getFSUserImportFilePath } from '@server/lib/paths.js' +import { move } from 'fs-extra/esm' +import { JobQueue } from '@server/lib/job-queue/job-queue.js' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' + +const userImportRouter = express.Router() + +userImportRouter.use(apiRateLimiter) + +userImportRouter.post('/:userId/imports/import-resumable', + authenticate, + asyncMiddleware(userImportRequestResumableInitValidator), + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +userImportRouter.delete('/:userId/imports/import-resumable', + authenticate, + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +userImportRouter.put('/:userId/imports/import-resumable', + authenticate, + uploadx.upload, // uploadx doesn't next() before the file upload completes + asyncMiddleware(userImportRequestResumableValidator), + asyncMiddleware(addUserImportResumable) +) + +userImportRouter.get('/:userId/imports/latest', + authenticate, + asyncMiddleware(getLatestImportStatusValidator), + asyncMiddleware(getLatestImport) +) + +// --------------------------------------------------------------------------- + +export { + userImportRouter +} + +// --------------------------------------------------------------------------- + +async function addUserImportResumable (req: express.Request, res: express.Response) { + const file = res.locals.importUserFileResumable + const user = res.locals.user + + // Move import + const userImport = new UserImportModel({ + state: UserImportState.PENDING, + userId: user.id, + createdAt: new Date() + }) + userImport.generateAndSetFilename() + + await move(file.path, getFSUserImportFilePath(userImport)) + + await saveInTransactionWithRetries(userImport) + + // Create job + await JobQueue.Instance.createJob({ type: 'import-user-archive', payload: { userImportId: userImport.id } }) + + logger.info('User import request job created for user ' + user.username) + + return res.json({ + userImport: { + id: userImport.id + } + } as UserImportUploadResult) +} + +async function getLatestImport (req: express.Request, res: express.Response) { + const userImport = await UserImportModel.loadLatestByUserId(res.locals.user.id) + if (!userImport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + + return res.json(userImport.toFormattedJSON()) +} diff --git a/server/core/controllers/api/video-channel.ts b/server/core/controllers/api/video-channel.ts index 4add1a637..bd0bf76f9 100644 --- a/server/core/controllers/api/video-channel.ts +++ b/server/core/controllers/api/video-channel.ts @@ -213,7 +213,12 @@ async function updateVideoChannelBanner (req: express.Request, res: express.Resp const videoChannel = res.locals.videoChannel const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) - const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER) + const banners = await updateLocalActorImageFiles({ + accountOrChannel: videoChannel, + imagePhysicalFile: bannerPhysicalFile, + type: ActorImageType.BANNER, + sendActorUpdate: true + }) auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) @@ -227,7 +232,13 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp const videoChannel = res.locals.videoChannel const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) - const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR) + const avatars = await updateLocalActorImageFiles({ + accountOrChannel: videoChannel, + imagePhysicalFile: avatarPhysicalFile, + type: ActorImageType.AVATAR, + sendActorUpdate: true + }) + auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) return res.json({ diff --git a/server/core/controllers/api/video-playlist.ts b/server/core/controllers/api/video-playlist.ts index 64207f50d..0b41a8c70 100644 --- a/server/core/controllers/api/video-playlist.ts +++ b/server/core/controllers/api/video-playlist.ts @@ -192,7 +192,6 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) { const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull if (thumbnailModel) { - thumbnailModel.automaticallyGenerated = false await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t) } diff --git a/server/core/controllers/api/videos/rate.ts b/server/core/controllers/api/videos/rate.ts index b0c4a328d..a75fc7f79 100644 --- a/server/core/controllers/api/videos/rate.ts +++ b/server/core/controllers/api/videos/rate.ts @@ -1,12 +1,8 @@ import express from 'express' import { HttpStatusCode, UserVideoRateUpdate } from '@peertube/peertube-models' import { logger } from '../../../helpers/logger.js' -import { VIDEO_RATE_TYPES } from '../../../initializers/constants.js' -import { sequelizeTypescript } from '../../../initializers/database.js' -import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates.js' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares/index.js' -import { AccountModel } from '../../../models/account/account.js' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { userRateVideo } from '@server/lib/rate.js' const rateVideoRouter = express.Router() @@ -25,63 +21,16 @@ export { // --------------------------------------------------------------------------- async function rateVideo (req: express.Request, res: express.Response) { - const body: UserVideoRateUpdate = req.body - const rateType = body.rating - const videoInstance = res.locals.videoAll - const userAccount = res.locals.oauth.token.User.Account + const user = res.locals.oauth.token.User + const video = res.locals.videoAll - await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const accountInstance = await AccountModel.load(userAccount.id, t) - const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) - - // Same rate, nothing do to - if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return - - let likesToIncrement = 0 - let dislikesToIncrement = 0 - - if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++ - else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ - - // There was a previous rate, update it - if (previousRate) { - // We will remove the previous rate, so we will need to update the video count attribute - if (previousRate.type === 'like') likesToIncrement-- - else if (previousRate.type === 'dislike') dislikesToIncrement-- - - if (rateType === 'none') { // Destroy previous rate - await previousRate.destroy(sequelizeOptions) - } else { // Update previous rate - previousRate.type = rateType - previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance) - await previousRate.save(sequelizeOptions) - } - } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate - const query = { - accountId: accountInstance.id, - videoId: videoInstance.id, - type: rateType, - url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance) - } - - await AccountVideoRateModel.create(query, sequelizeOptions) - } - - const incrementQuery = { - likes: likesToIncrement, - dislikes: dislikesToIncrement - } - - await videoInstance.increment(incrementQuery, sequelizeOptions) - - await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) - - logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) + await userRateVideo({ + account: user.Account, + rateType: (req.body as UserVideoRateUpdate).rating, + video }) - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() + logger.info('Account video rate for video %s of account %s updated.', video.name, user.username) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts index d56e9610f..b0172c56d 100644 --- a/server/core/controllers/api/videos/source.ts +++ b/server/core/controllers/api/videos/source.ts @@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q import { Hooks } from '@server/lib/plugins/hooks.js' import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' import { uploadx } from '@server/lib/uploadx.js' -import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js' +import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' import { buildNewFile } from '@server/lib/video-file.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' diff --git a/server/core/controllers/api/videos/update.ts b/server/core/controllers/api/videos/update.ts index 2e168b029..600e005df 100644 --- a/server/core/controllers/api/videos/update.ts +++ b/server/core/controllers/api/videos/update.ts @@ -6,7 +6,7 @@ import { exists } from '@server/helpers/custom-validators/misc.js' import { changeVideoChannelShare } from '@server/lib/activitypub/share.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { setVideoPrivacy } from '@server/lib/video-privacy.js' -import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' +import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' import { openapiOperationDoc } from '@server/middlewares/doc.js' import { VideoPasswordModel } from '@server/models/video/video-password.js' import { FilteredModelAttributes } from '@server/types/index.js' @@ -23,6 +23,7 @@ import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosU import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js' import { VideoModel } from '../../../models/video/video.js' import { replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js' +import { addVideoJobsAfterUpdate } from '@server/lib/video-jobs.js' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') diff --git a/server/core/controllers/api/videos/upload.ts b/server/core/controllers/api/videos/upload.ts index 75f6eb203..9c5966313 100644 --- a/server/core/controllers/api/videos/upload.ts +++ b/server/core/controllers/api/videos/upload.ts @@ -3,14 +3,10 @@ import { move } from 'fs-extra/esm' import { basename } from 'path' import { getResumableUploadPath } from '@server/helpers/upload.js' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' -import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js' import { Redis } from '@server/lib/redis.js' import { uploadx } from '@server/lib/uploadx.js' import { - buildLocalVideoFromReq, - buildMoveJob, - buildStoryboardJobIfNeeded, - buildVideoThumbnailsFromReq, + buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' import { buildNewFile } from '@server/lib/video-file.js' @@ -21,7 +17,7 @@ import { VideoPasswordModel } from '@server/models/video/video-password.js' import { VideoSourceModel } from '@server/models/video/video-source.js' import { MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js' import { uuidToShort } from '@peertube/peertube-node-utils' -import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@peertube/peertube-models' +import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy } from '@peertube/peertube-models' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js' import { createReqFiles } from '../../../helpers/express-utils.js' import { logger, loggerTagsFactory } from '../../../helpers/logger.js' @@ -43,6 +39,7 @@ import { VideoModel } from '../../../models/video/video.js' import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg' import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js' import { FfprobeData } from 'fluent-ffmpeg' +import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -230,7 +227,7 @@ async function addVideo (options: { // Channel has a new content, set as updated await videoCreated.VideoChannel.setAsUpdated() - addVideoJobsAfterUpload(videoCreated, videoFile) + addVideoJobsAfterCreation({ video: videoCreated, videoFile }) .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res }) @@ -244,55 +241,6 @@ async function addVideo (options: { } } -async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { - const jobs: (CreateJobArgument & CreateJobOptions)[] = [ - { - type: 'manage-video-torrent' as 'manage-video-torrent', - payload: { - videoId: video.id, - videoFileId: videoFile.id, - action: 'create' - } - }, - - buildStoryboardJobIfNeeded({ video, federate: false }), - - { - type: 'notify', - payload: { - action: 'new-video', - videoUUID: video.uuid - } - }, - - { - type: 'federate-video' as 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideoForFederation: true - } - } - ] - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' })) - } - - if (video.state === VideoState.TO_TRANSCODE) { - jobs.push({ - type: 'transcoding-job-builder' as 'transcoding-job-builder', - payload: { - videoUUID: video.uuid, - optimizeJob: { - isNewVideo: true - } - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} - async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { await Redis.Instance.deleteUploadSession(req.query.upload_id) diff --git a/server/core/controllers/download.ts b/server/core/controllers/download.ts index ca28eee44..f653014b9 100644 --- a/server/core/controllers/download.ts +++ b/server/core/controllers/download.ts @@ -2,14 +2,30 @@ import cors from 'cors' import express from 'express' import { logger } from '@server/helpers/logger.js' import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js' -import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js' +import { + generateHLSFilePresignedUrl, + generateUserExportPresignedUrl, + generateWebVideoPresignedUrl +} from '@server/lib/object-storage/index.js' import { Hooks } from '@server/lib/plugins/hooks.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { + MStreamingPlaylist, + MStreamingPlaylistVideo, + MUserExport, + MVideo, + MVideoFile, + MVideoFullLight +} from '@server/types/models/index.js' import { forceNumber } from '@peertube/peertube-core-utils' -import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js' -import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares/index.js' +import { + asyncMiddleware, optionalAuthenticate, + userExportDownloadValidator, + videosDownloadValidator +} from '../middlewares/index.js' +import { getFSUserExportFilePath } from '@server/lib/paths.js' const downloadRouter = express.Router() @@ -34,6 +50,12 @@ downloadRouter.use( asyncMiddleware(downloadHLSVideoFile) ) +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename', + asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication + asyncMiddleware(downloadUserExport) +) + // --------------------------------------------------------------------------- export { @@ -99,8 +121,8 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { const videoName = video.name.replace(/[/\\]/g, '_') const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}` - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename }) + if (videoFile.storage === FileStorage.OBJECT_STORAGE) { + return redirectVideoDownloadToObjectStorage({ res, video, file: videoFile, downloadFilename }) } await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { @@ -140,8 +162,8 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response const videoName = video.name.replace(/\//g, '_') const downloadFilename = `${videoName}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename }) + if (videoFile.storage === FileStorage.OBJECT_STORAGE) { + return redirectVideoDownloadToObjectStorage({ res, video, streamingPlaylist, file: videoFile, downloadFilename }) } await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { @@ -149,6 +171,21 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response }) } +function downloadUserExport (req: express.Request, res: express.Response) { + const userExport = res.locals.userExport + + const downloadFilename = userExport.filename + + if (userExport.storage === FileStorage.OBJECT_STORAGE) { + return redirectUserExportToObjectStorage({ res, userExport, downloadFilename }) + } + + res.download(getFSUserExportFilePath(userExport), downloadFilename) + return Promise.resolve() +} + +// --------------------------------------------------------------------------- + function getVideoFile (req: express.Request, files: MVideoFile[]) { const resolution = forceNumber(req.params.resolution) return files.find(f => f.resolution === resolution) @@ -194,8 +231,7 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?: return true } -async function redirectToObjectStorage (options: { - req: express.Request +async function redirectVideoDownloadToObjectStorage (options: { res: express.Response video: MVideo file: MVideoFile @@ -212,3 +248,17 @@ async function redirectToObjectStorage (options: { return res.redirect(url) } + +async function redirectUserExportToObjectStorage (options: { + res: express.Response + downloadFilename: string + userExport: MUserExport +}) { + const { res, downloadFilename, userExport } = options + + const url = await generateUserExportPresignedUrl({ userExport, downloadFilename }) + + logger.debug('Generating pre-signed URL %s for user export %s', url, userExport.filename) + + return res.redirect(url) +} diff --git a/server/core/helpers/captions-utils.ts b/server/core/helpers/captions-utils.ts index f165cb447..97110268f 100644 --- a/server/core/helpers/captions-utils.ts +++ b/server/core/helpers/captions-utils.ts @@ -1,14 +1,11 @@ import { createReadStream, createWriteStream } from 'fs' import { move, remove } from 'fs-extra/esm' -import { join } from 'path' import { Transform } from 'stream' import { MVideoCaption } from '@server/types/models/index.js' -import { CONFIG } from '../initializers/config.js' import { pipelinePromise } from './core-utils.js' -async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) { - const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR - const destination = join(videoCaptionsDir, videoCaption.filename) +async function moveAndProcessCaptionFile (physicalFile: { filename?: string, path: string }, videoCaption: MVideoCaption) { + const destination = videoCaption.getFSPath() // Convert this srt file to vtt if (physicalFile.path.endsWith('.srt')) { @@ -19,7 +16,7 @@ async function moveAndProcessCaptionFile (physicalFile: { filename: string, path } // This is important in case if there is another attempt in the retry process - physicalFile.filename = videoCaption.filename + if (physicalFile.filename) physicalFile.filename = videoCaption.filename physicalFile.path = destination } diff --git a/server/core/helpers/unzip.ts b/server/core/helpers/unzip.ts new file mode 100644 index 000000000..8a892be7c --- /dev/null +++ b/server/core/helpers/unzip.ts @@ -0,0 +1,55 @@ +import { createWriteStream } from 'fs' +import { ensureDir } from 'fs-extra/esm' +import { dirname, join } from 'path' +import { pipeline } from 'stream' +import * as yauzl from 'yauzl' +import { logger, loggerTagsFactory } from './logger.js' + +const lTags = loggerTagsFactory('unzip') + +export async function unzip (source: string, destination: string) { + await ensureDir(destination) + + logger.info(`Unzip ${source} to ${destination}`, lTags()) + + return new Promise((res, rej) => { + yauzl.open(source, { lazyEntries: true }, (err, zipFile) => { + if (err) return rej(err) + + zipFile.readEntry() + + zipFile.on('entry', async entry => { + const entryPath = join(destination, entry.fileName) + + try { + if (/\/$/.test(entry.fileName)) { + await ensureDir(entryPath) + logger.debug(`Creating directory from zip ${entryPath}`, lTags()) + + zipFile.readEntry() + return + } + + await ensureDir(dirname(entryPath)) + } catch (err) { + return rej(err) + } + + zipFile.openReadStream(entry, (readErr, readStream) => { + if (readErr) return rej(readErr) + + logger.debug(`Creating file from zip ${entryPath}`, lTags()) + + const writeStream = createWriteStream(entryPath) + writeStream.on('close', () => zipFile.readEntry()) + + pipeline(readStream, writeStream, pipelineErr => { + if (pipelineErr) return rej(pipelineErr) + }) + }) + }) + + zipFile.on('end', () => res()) + }) + }) +} diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index c79501a1d..99e120ff6 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -153,6 +153,11 @@ const CONFIG = { BUCKET_NAME: config.get('object_storage.streaming_playlists.bucket_name'), PREFIX: config.get('object_storage.streaming_playlists.prefix'), BASE_URL: config.get('object_storage.streaming_playlists.base_url') + }, + USER_EXPORTS: { + BUCKET_NAME: config.get('object_storage.user_exports.bucket_name'), + PREFIX: config.get('object_storage.user_exports.prefix'), + BASE_URL: config.get('object_storage.user_exports.base_url') } }, WEBSERVER: { @@ -511,6 +516,16 @@ const CONFIG = { get FULL_SYNC_VIDEOS_LIMIT () { return config.get('import.video_channel_synchronization.full_sync_videos_limit') } + }, + USERS: { + get ENABLED () { return config.get('import.users.enabled') } + } + }, + EXPORT: { + USERS: { + get ENABLED () { return config.get('export.users.enabled') }, + get MAX_USER_VIDEO_QUOTA () { return parseBytes(config.get('export.users.max_user_video_quota')) }, + get EXPORT_EXPIRATION () { return parseDurationToMs(config.get('export.users.export_expiration')) } } }, AUTO_BLACKLIST: { diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index f436451e3..2fee8ff3e 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -10,6 +10,10 @@ import { NSFWPolicyType, RunnerJobState, RunnerJobStateType, + UserExportState, + UserExportStateType, + UserImportState, + UserImportStateType, UserRegistrationState, UserRegistrationStateType, VideoChannelSyncState, @@ -41,7 +45,7 @@ import { cpus } from 'os' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 805 +const LAST_MIGRATION_VERSION = 815 // --------------------------------------------------------------------------- @@ -191,7 +195,9 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'transcoding-job-builder': 1, 'generate-video-storyboard': 1, 'notify': 1, - 'federate-video': 1 + 'federate-video': 1, + 'create-user-export': 1, + 'import-user-archive': 1 } // Excluded keys are jobs that can be configured by admins const JOB_CONCURRENCY: { [id in Exclude]: number } = { @@ -217,7 +223,9 @@ const JOB_CONCURRENCY: { [id in Exclude { + const query = ` + CREATE TABLE IF NOT EXISTS "userExport" ( + "id" SERIAL, + "filename" VARCHAR(255), + "withVideoFiles" BOOLEAN NOT NULL, + "state" INTEGER NOT NULL, + "error" TEXT, + "size" INTEGER, + "storage" INTEGER NOT NULL, + "userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, + "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY ("id") + );` + + await utils.sequelize.query(query, { transaction: utils.transaction }) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/core/initializers/migrations/0815-user-import.ts b/server/core/initializers/migrations/0815-user-import.ts new file mode 100644 index 000000000..531ec9847 --- /dev/null +++ b/server/core/initializers/migrations/0815-user-import.ts @@ -0,0 +1,31 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const query = ` + CREATE TABLE IF NOT EXISTS "userImport" ( + "id" SERIAL, + "filename" VARCHAR(255), + "state" INTEGER NOT NULL, + "error" TEXT, + "resultSummary" JSONB, + "userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, + "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY ("id") + );;` + + await utils.sequelize.query(query, { transaction: utils.transaction }) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/core/lib/activitypub/collection.ts b/server/core/lib/activitypub/collection.ts index 280c3a5e9..8856b7896 100644 --- a/server/core/lib/activitypub/collection.ts +++ b/server/core/lib/activitypub/collection.ts @@ -7,7 +7,7 @@ import { forceNumber } from '@peertube/peertube-core-utils' type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird> | Promise> -async function activityPubCollectionPagination ( +export async function activityPubCollectionPagination ( baseUrl: string, handler: ActivityPubCollectionPaginationHandler, page?: any, @@ -56,8 +56,11 @@ async function activityPubCollectionPagination ( } } -// --------------------------------------------------------------------------- - -export { - activityPubCollectionPagination +export function activityPubCollection (baseUrl: string, items: T[]) { + return { + id: baseUrl, + type: 'OrderedCollection' as 'OrderedCollection', + totalItems: items.length, + orderedItems: items + } } diff --git a/server/core/lib/activitypub/process/process-delete.ts b/server/core/lib/activitypub/process/process-delete.ts index 932e4cf94..6c4f3099a 100644 --- a/server/core/lib/activitypub/process/process-delete.ts +++ b/server/core/lib/activitypub/process/process-delete.ts @@ -51,7 +51,7 @@ async function processDeleteActivity (options: APProcessorOptions { - const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t) + const video = await VideoModel.loadByUrlAndPopulateAccountAndFiles(uri, t) let videoComment: MCommentOwnerVideo let flaggedAccount: MAccountDefault diff --git a/server/core/lib/activitypub/videos/refresh.ts b/server/core/lib/activitypub/videos/refresh.ts index 961bdf71c..59a77ba24 100644 --- a/server/core/lib/activitypub/videos/refresh.ts +++ b/server/core/lib/activitypub/videos/refresh.ts @@ -18,7 +18,7 @@ async function refreshVideoIfNeeded (options: { // We need more attributes if the argument video was fetched with not enough joints const video = options.fetchedType === 'all' ? options.video as MVideoAccountLightBlacklistAllFiles - : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + : await VideoModel.loadByUrlAndPopulateAccountAndFiles(options.video.url) const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) diff --git a/server/core/lib/blocklist.ts b/server/core/lib/blocklist.ts index 7491d92d7..fd0e9f6fc 100644 --- a/server/core/lib/blocklist.ts +++ b/server/core/lib/blocklist.ts @@ -3,23 +3,51 @@ import { getServerActor } from '@server/models/application/application.js' import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models/index.js' import { AccountBlocklistModel } from '../models/account/account-blocklist.js' import { ServerBlocklistModel } from '../models/server/server-blocklist.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { logger } from '@server/helpers/logger.js' -function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { - return sequelizeTypescript.transaction(async t => { +async function addAccountInBlocklist (options: { + byAccountId: number + targetAccountId: number + + removeNotificationOfUserId: number | null // If blocked by a user +}) { + const { byAccountId, targetAccountId, removeNotificationOfUserId } = options + + await sequelizeTypescript.transaction(async t => { return AccountBlocklistModel.upsert({ accountId: byAccountId, targetAccountId }, { transaction: t }) }) + + UserNotificationModel.removeNotificationsOf({ + id: targetAccountId, + type: 'account', + forUserId: removeNotificationOfUserId + }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) } -function addServerInBlocklist (byAccountId: number, targetServerId: number) { - return sequelizeTypescript.transaction(async t => { +async function addServerInBlocklist (options: { + byAccountId: number + targetServerId: number + + removeNotificationOfUserId: number | null +}) { + const { byAccountId, targetServerId, removeNotificationOfUserId } = options + + await sequelizeTypescript.transaction(async t => { return ServerBlocklistModel.upsert({ accountId: byAccountId, targetServerId }, { transaction: t }) }) + + UserNotificationModel.removeNotificationsOf({ + id: targetServerId, + type: 'server', + forUserId: removeNotificationOfUserId + }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) } function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) { diff --git a/server/core/lib/emailer.ts b/server/core/lib/emailer.ts index bd66e9450..846b5d984 100644 --- a/server/core/lib/emailer.ts +++ b/server/core/lib/emailer.ts @@ -1,5 +1,5 @@ import { arrayify } from '@peertube/peertube-core-utils' -import { EmailPayload, SendEmailDefaultOptions, UserRegistrationState } from '@peertube/peertube-models' +import { EmailPayload, SendEmailDefaultOptions, UserExportState, UserRegistrationState } from '@peertube/peertube-models' import { isTestOrDevInstance, root } from '@peertube/peertube-node-utils' import { readFileSync } from 'fs' import merge from 'lodash-es/merge.js' @@ -8,8 +8,9 @@ import { join } from 'path' import { bunyanLogger, logger } from '../helpers/logger.js' import { CONFIG, isEmailEnabled } from '../initializers/config.js' import { WEBSERVER } from '../initializers/constants.js' -import { MRegistration, MUser } from '../types/models/index.js' +import { MRegistration, MUser, MUserExport, MUserImport } from '../types/models/index.js' import { JobQueue } from './job-queue/index.js' +import { UserModel } from '@server/models/user/user.js' class Emailer { @@ -52,6 +53,8 @@ class Emailer { } } + // --------------------------------------------------------------------------- + addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) { const emailPayload: EmailPayload = { template: 'password-reset', @@ -160,13 +163,82 @@ class Emailer { locals: { username: registration.username, moderationResponse: registration.moderationResponse, - loginLink: WEBSERVER.URL + '/login' + loginLink: WEBSERVER.URL + '/login', + + hideNotificationPreferencesLink: true } } return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) } + // --------------------------------------------------------------------------- + + async addUserExportCompletedOrErroredJob (userExport: MUserExport) { + let template: string + let subject: string + + if (userExport.state === UserExportState.COMPLETED) { + template = 'user-export-completed' + subject = `Your export archive has been created` + } else { + template = 'user-export-errored' + subject = `Failed to create your export archive` + } + + const user = await UserModel.loadById(userExport.userId) + + const emailPayload: EmailPayload = { + to: [ user.email ], + template, + subject, + locals: { + exportsUrl: WEBSERVER.URL + '/my-account/import-export', + errorMessage: userExport.error, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + async addUserImportErroredJob (userImport: MUserImport) { + const user = await UserModel.loadById(userImport.userId) + + const emailPayload: EmailPayload = { + to: [ user.email ], + template: 'user-import-errored', + subject: 'Failed to import your archive', + locals: { + errorMessage: userImport.error, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + async addUserImportSuccessJob (userImport: MUserImport) { + const user = await UserModel.loadById(userImport.userId) + + const emailPayload: EmailPayload = { + to: [ user.email ], + template: 'user-import-completed', + subject: 'Your archive import has finished', + locals: { + resultStats: userImport.resultSummary.stats, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + // --------------------------------------------------------------------------- + async sendMail (options: EmailPayload) { if (!isEmailEnabled()) { logger.info('Cannot send mail because SMTP is not configured.') @@ -233,14 +305,14 @@ class Emailer { private initSMTPTransport () { logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) - let tls + let tls: { ca: [ Buffer ] } if (CONFIG.SMTP.CA_FILE) { tls = { ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ] } } - let auth + let auth: { user: string, pass: string } if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) { auth = { user: CONFIG.SMTP.USERNAME, diff --git a/server/core/lib/emails/user-export-completed/html.pug b/server/core/lib/emails/user-export-completed/html.pug new file mode 100644 index 000000000..00cf64211 --- /dev/null +++ b/server/core/lib/emails/user-export-completed/html.pug @@ -0,0 +1,9 @@ +extends ../common/greetings +include ../common/mixins.pug + +block title + | Your export archive has been created + +block content + p + | Your export archive has been created. You can download it in #[a(href=exportsUrl) your account export page]. diff --git a/server/core/lib/emails/user-export-errored/html.pug b/server/core/lib/emails/user-export-errored/html.pug new file mode 100644 index 000000000..7699a5dd4 --- /dev/null +++ b/server/core/lib/emails/user-export-errored/html.pug @@ -0,0 +1,12 @@ +extends ../common/greetings +include ../common/mixins.pug + +block title + | Failed to create your export archive + +block content + p + | We are sorry but the generation of your export archive has failed: + blockquote !{errorMessage} + p + | Please contact your administrator if the problem occurs again. diff --git a/server/core/lib/emails/user-import-completed/html.pug b/server/core/lib/emails/user-import-completed/html.pug new file mode 100644 index 000000000..7449b6743 --- /dev/null +++ b/server/core/lib/emails/user-import-completed/html.pug @@ -0,0 +1,46 @@ +extends ../common/greetings +include ../common/mixins.pug + +mixin displaySummary(stats) + ul + if stats.success + li Imported: #{stats.success} + if stats.duplicates + li Not imported as considered duplicate: #{stats.duplicates} + if stats.errors + li Not imported due to error: #{stats.errors} + +block title + | Your archive import has finished + +block content + p Your archive import has finished. Here is the summary of imported objects: + + ul + li + strong User settings: + +displaySummary(resultStats.userSettings) + li + strong Account (name, description, avatar...): + +displaySummary(resultStats.account) + li + strong Blocklist: + +displaySummary(resultStats.blocklist) + li + strong Channels: + +displaySummary(resultStats.channels) + li + strong Likes: + +displaySummary(resultStats.likes) + li + strong Dislikes: + +displaySummary(resultStats.dislikes) + li + strong Subscriptions: + +displaySummary(resultStats.following) + li + strong Video Playlists: + +displaySummary(resultStats.videoPlaylists) + li + strong Videos: + +displaySummary(resultStats.videos) diff --git a/server/core/lib/emails/user-import-errored/html.pug b/server/core/lib/emails/user-import-errored/html.pug new file mode 100644 index 000000000..8f5041d1e --- /dev/null +++ b/server/core/lib/emails/user-import-errored/html.pug @@ -0,0 +1,12 @@ +extends ../common/greetings +include ../common/mixins.pug + +block title + | Failed to import your archive + +block content + p + | We are sorry but the import of your archive has failed: + blockquote !{errorMessage} + p + | Please contact your administrator if the problem occurs again. diff --git a/server/core/lib/files-cache/video-captions-simple-file-cache.ts b/server/core/lib/files-cache/video-captions-simple-file-cache.ts index 26853706e..5f7e5a158 100644 --- a/server/core/lib/files-cache/video-captions-simple-file-cache.ts +++ b/server/core/lib/files-cache/video-captions-simple-file-cache.ts @@ -1,7 +1,6 @@ import { join } from 'path' import { logger } from '@server/helpers/logger.js' import { doRequestAndSaveToFile } from '@server/helpers/requests.js' -import { CONFIG } from '../../initializers/config.js' import { FILES_CACHE } from '../../initializers/constants.js' import { VideoModel } from '../../models/video/video.js' import { VideoCaptionModel } from '../../models/video/video-caption.js' @@ -24,7 +23,7 @@ class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache { if (!videoCaption) return undefined if (videoCaption.isOwned()) { - return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } + return { isOwned: true, path: videoCaption.getFSPath() } } return this.loadRemoteFile(filename) diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.ts index 9a53593fb..a2d9ca024 100644 --- a/server/core/lib/hls.ts +++ b/server/core/lib/hls.ts @@ -1,6 +1,6 @@ import { uniqify, uuidRegex } from '@peertube/peertube-core-utils' import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg' -import { VideoStorage } from '@peertube/peertube-models' +import { FileStorage } from '@peertube/peertube-models' import { sha256 } from '@peertube/peertube-node-utils' import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models/index.js' import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm' @@ -100,7 +100,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + if (playlist.storage === FileStorage.OBJECT_STORAGE) { playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) await remove(masterPlaylistPath) } @@ -151,7 +151,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) await outputJSON(outputPath, json) - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + if (playlist.storage === FileStorage.OBJECT_STORAGE) { playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) await remove(outputPath) } diff --git a/server/core/lib/job-queue/handlers/activitypub-follow.ts b/server/core/lib/job-queue/handlers/activitypub-follow.ts index 0fb322345..02c85cfe4 100644 --- a/server/core/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/core/lib/job-queue/handlers/activitypub-follow.ts @@ -18,6 +18,10 @@ async function processActivityPubFollow (job: Job) { const payload = job.data as ActivitypubFollowPayload const host = payload.host + const handle = host + ? `${payload.name}@${host}` + : payload.name + logger.info('Processing ActivityPub follow in job %s.', job.id) let targetActor: MActorFull @@ -30,14 +34,24 @@ async function processActivityPubFollow (job: Job) { let actorUrl: string - if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost) - if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost) + try { + if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost) + if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost) - targetActor = await getOrCreateAPActor(actorUrl, 'all') + targetActor = await getOrCreateAPActor(actorUrl, 'all') + } catch (err) { + logger.warn(`Do not follow ${handle} because we could not find the actor URL (in database or using webfinger)`) + return + } + } + + if (!targetActor) { + logger.warn(`Do not follow ${handle} because we could not fetch/load the actor`) + return } if (payload.assertIsChannel && !targetActor.VideoChannel) { - logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host) + logger.warn(`Do not follow ${handle} because it is not a channel.`) return } diff --git a/server/core/lib/job-queue/handlers/create-user-export.ts b/server/core/lib/job-queue/handlers/create-user-export.ts new file mode 100644 index 000000000..0c1bc07fe --- /dev/null +++ b/server/core/lib/job-queue/handlers/create-user-export.ts @@ -0,0 +1,34 @@ +import { Job } from 'bullmq' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CreateUserExportPayload } from '@peertube/peertube-models' +import { UserExportModel } from '@server/models/user/user-export.js' +import { UserExporter } from '@server/lib/user-import-export/user-exporter.js' +import { Emailer } from '@server/lib/emailer.js' + +const lTags = loggerTagsFactory('user-export') + +export async function processCreateUserExport (job: Job): Promise { + const payload = job.data as CreateUserExportPayload + const exportModel = await UserExportModel.load(payload.userExportId) + + logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags()) + + if (!exportModel) { + logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags()) + return + } + + const exporter = new UserExporter() + + try { + await exporter.export(exportModel) + + await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel) + + logger.info(`User export ${payload.userExportId} has been created`, lTags()) + } catch (err) { + await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel) + + throw err + } +} diff --git a/server/core/lib/job-queue/handlers/import-user-archive.ts b/server/core/lib/job-queue/handlers/import-user-archive.ts new file mode 100644 index 000000000..7a6d4b2c3 --- /dev/null +++ b/server/core/lib/job-queue/handlers/import-user-archive.ts @@ -0,0 +1,33 @@ +import { Job } from 'bullmq' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { ImportUserArchivePayload } from '@peertube/peertube-models' +import { UserImportModel } from '@server/models/user/user-import.js' +import { UserImporter } from '@server/lib/user-import-export/user-importer.js' +import { Emailer } from '@server/lib/emailer.js' + +const lTags = loggerTagsFactory('user-import') + +export async function processImportUserArchive (job: Job): Promise { + const payload = job.data as ImportUserArchivePayload + const importModel = await UserImportModel.load(payload.userImportId) + + logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags()) + + if (!importModel) { + logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags()) + return + } + + const exporter = new UserImporter() + await exporter.import(importModel) + + try { + await Emailer.Instance.addUserImportSuccessJob(importModel) + + logger.info(`User import ${payload.userImportId} ended`, lTags()) + } catch (err) { + await Emailer.Instance.addUserImportErroredJob(importModel) + + throw err + } +} diff --git a/server/core/lib/job-queue/handlers/move-to-file-system.ts b/server/core/lib/job-queue/handlers/move-to-file-system.ts index d21f200da..1925db4aa 100644 --- a/server/core/lib/job-queue/handlers/move-to-file-system.ts +++ b/server/core/lib/job-queue/handlers/move-to-file-system.ts @@ -1,6 +1,6 @@ import { Job } from 'bullmq' import { join } from 'path' -import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models' +import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' @@ -52,7 +52,7 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) { async function moveWebVideoFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { - if (file.storage === VideoStorage.FILE_SYSTEM) continue + if (file.storage === FileStorage.FILE_SYSTEM) continue await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)) await onFileMoved({ @@ -68,7 +68,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { const playlistWithVideo = playlist.withVideo(video) for (const file of playlist.VideoFiles) { - if (file.storage === VideoStorage.FILE_SYSTEM) continue + if (file.storage === FileStorage.FILE_SYSTEM) continue // Resolution playlist const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) @@ -97,7 +97,7 @@ async function onFileMoved (options: { const oldFileUrl = file.fileUrl file.fileUrl = null - file.storage = VideoStorage.FILE_SYSTEM + file.storage = FileStorage.FILE_SYSTEM await updateTorrentMetadata(videoOrPlaylist, file) await file.save() @@ -114,7 +114,7 @@ async function doAfterLastMove (options: { const { video, previousVideoState, isNewVideo } = options for (const playlist of video.VideoStreamingPlaylists) { - if (playlist.storage === VideoStorage.FILE_SYSTEM) continue + if (playlist.storage === FileStorage.FILE_SYSTEM) continue const playlistWithVideo = playlist.withVideo(video) @@ -124,7 +124,7 @@ async function doAfterLastMove (options: { playlist.playlistUrl = null playlist.segmentsSha256Url = null - playlist.storage = VideoStorage.FILE_SYSTEM + playlist.storage = FileStorage.FILE_SYSTEM playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION diff --git a/server/core/lib/job-queue/handlers/move-to-object-storage.ts b/server/core/lib/job-queue/handlers/move-to-object-storage.ts index 6dccc897b..7057dd0ca 100644 --- a/server/core/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/core/lib/job-queue/handlers/move-to-object-storage.ts @@ -1,7 +1,7 @@ import { Job } from 'bullmq' import { remove } from 'fs-extra/esm' import { join } from 'path' -import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models' +import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' @@ -45,7 +45,7 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) { async function moveWebVideoFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue + if (file.storage !== FileStorage.FILE_SYSTEM) continue const fileUrl = await storeWebVideoFile(video, file) @@ -59,7 +59,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { const playlistWithVideo = playlist.withVideo(video) for (const file of playlist.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue + if (file.storage !== FileStorage.FILE_SYSTEM) continue // Resolution playlist const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) @@ -84,7 +84,7 @@ async function onFileMoved (options: { const { videoOrPlaylist, file, fileUrl, oldPath } = options file.fileUrl = fileUrl - file.storage = VideoStorage.OBJECT_STORAGE + file.storage = FileStorage.OBJECT_STORAGE await updateTorrentMetadata(videoOrPlaylist, file) await file.save() @@ -101,13 +101,13 @@ async function doAfterLastMove (options: { const { video, previousVideoState, isNewVideo } = options for (const playlist of video.VideoStreamingPlaylists) { - if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue + if (playlist.storage === FileStorage.OBJECT_STORAGE) continue const playlistWithVideo = playlist.withVideo(video) playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) - playlist.storage = VideoStorage.OBJECT_STORAGE + playlist.storage = FileStorage.OBJECT_STORAGE playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION diff --git a/server/core/lib/job-queue/handlers/video-file-import.ts b/server/core/lib/job-queue/handlers/video-file-import.ts index 64dc63ad9..a306c6b80 100644 --- a/server/core/lib/job-queue/handlers/video-file-import.ts +++ b/server/core/lib/job-queue/handlers/video-file-import.ts @@ -1,13 +1,12 @@ import { Job } from 'bullmq' import { copy } from 'fs-extra/esm' import { stat } from 'fs/promises' -import { VideoFileImportPayload, VideoStorage } from '@peertube/peertube-models' +import { VideoFileImportPayload, FileStorage } from '@peertube/peertube-models' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' import { CONFIG } from '@server/initializers/config.js' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' import { generateWebVideoFilename } from '@server/lib/paths.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' -import { buildMoveJob } from '@server/lib/video.js' import { VideoFileModel } from '@server/models/video/video-file.js' import { VideoModel } from '@server/models/video/video.js' import { MVideoFullLight } from '@server/types/models/index.js' @@ -15,6 +14,7 @@ import { getLowercaseExtension } from '@peertube/peertube-node-utils' import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' import { logger } from '../../../helpers/logger.js' import { JobQueue } from '../job-queue.js' +import { buildMoveJob } from '@server/lib/video-jobs.js' async function processVideoFileImport (job: Job) { const payload = job.data as VideoFileImportPayload @@ -68,7 +68,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { resolution, extname: fileExt, filename: generateWebVideoFilename(resolution, fileExt), - storage: VideoStorage.FILE_SYSTEM, + storage: FileStorage.FILE_SYSTEM, size, fps, videoId: video.id diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts index dfa0ca7dd..6ebe973db 100644 --- a/server/core/lib/job-queue/handlers/video-import.ts +++ b/server/core/lib/job-queue/handlers/video-import.ts @@ -22,10 +22,10 @@ import { generateWebVideoFilename } from '@server/lib/paths.js' import { Hooks } from '@server/lib/plugins/hooks.js' import { ServerConfigManager } from '@server/lib/server-config-manager.js' import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js' -import { isAbleToUploadVideo } from '@server/lib/user.js' +import { isUserQuotaValid } from '@server/lib/user.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { buildNextVideoState } from '@server/lib/video-state.js' -import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js' +import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js' import { getLowercaseExtension } from '@peertube/peertube-node-utils' @@ -138,7 +138,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid // Get information about this video const stats = await stat(tempVideoPath) - const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) + const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size }) if (isAble === false) { throw new Error('The user video quota is exceeded with this video to import.') } diff --git a/server/core/lib/job-queue/handlers/video-live-ending.ts b/server/core/lib/job-queue/handlers/video-live-ending.ts index eecb931c8..206ce2108 100644 --- a/server/core/lib/job-queue/handlers/video-live-ending.ts +++ b/server/core/lib/job-queue/handlers/video-live-ending.ts @@ -30,7 +30,7 @@ import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoS import { logger, loggerTagsFactory } from '../../../helpers/logger.js' import { JobQueue } from '../job-queue.js' import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js' -import { buildStoryboardJobIfNeeded } from '@server/lib/video.js' +import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' const lTags = loggerTagsFactory('live', 'job') diff --git a/server/core/lib/job-queue/handlers/video-studio-edition.ts b/server/core/lib/job-queue/handlers/video-studio-edition.ts index 3ddf0fa82..c0a1c60a6 100644 --- a/server/core/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/core/lib/job-queue/handlers/video-studio-edition.ts @@ -4,7 +4,7 @@ import { join } from 'path' import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js' import { CONFIG } from '@server/initializers/config.js' import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js' -import { isAbleToUploadVideo } from '@server/lib/user.js' +import { isUserQuotaValid } from '@server/lib/user.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js' import { UserModel } from '@server/models/user/user.js' @@ -170,7 +170,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder) - if (await isAbleToUploadVideo(user.id, additionalBytes) === false) { + if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) { throw new Error('Quota exceeded for this user to edit the video') } } diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts index f0515fd76..5559c8ea3 100644 --- a/server/core/lib/job-queue/job-queue.ts +++ b/server/core/lib/job-queue/job-queue.ts @@ -18,10 +18,12 @@ import { ActivitypubHttpUnicastPayload, ActorKeysPayload, AfterVideoChannelImportPayload, + CreateUserExportPayload, DeleteResumableUploadMetaFilePayload, EmailPayload, FederateVideoPayload, GenerateStoryboardPayload, + ImportUserArchivePayload, JobState, JobType, ManageVideoTorrentPayload, @@ -71,6 +73,8 @@ import { processVideoStudioEdition } from './handlers/video-studio-edition.js' import { processVideoTranscoding } from './handlers/video-transcoding.js' import { processVideosViewsStats } from './handlers/video-views-stats.js' import { onMoveToFileSystemFailure, processMoveToFileSystem } from './handlers/move-to-file-system.js' +import { processCreateUserExport } from './handlers/create-user-export.js' +import { processImportUserArchive } from './handlers/import-user-archive.js' export type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -98,7 +102,9 @@ export type CreateJobArgument = { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | { type: 'notify', payload: NotifyPayload } | { type: 'federate-video', payload: FederateVideoPayload } | - { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } + { type: 'create-user-export', payload: CreateUserExportPayload } | + { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } | + { type: 'import-user-archive', payload: ImportUserArchivePayload } export type CreateJobOptions = { delay?: number @@ -131,7 +137,9 @@ const handlers: { [id in JobType]: (job: Job) => Promise } = { 'video-studio-edition': processVideoStudioEdition, 'video-transcoding': processVideoTranscoding, 'videos-views-stats': processVideosViewsStats, - 'generate-video-storyboard': processGenerateStoryboard + 'generate-video-storyboard': processGenerateStoryboard, + 'create-user-export': processCreateUserExport, + 'import-user-archive': processImportUserArchive } const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { @@ -164,7 +172,9 @@ const jobTypes: JobType[] = [ 'video-redundancy', 'video-studio-edition', 'video-transcoding', - 'videos-views-stats' + 'videos-views-stats', + 'create-user-export', + 'import-user-archive' ] const silentFailure = new Set([ 'activitypub-http-unicast' ]) diff --git a/server/core/lib/live/live-utils.ts b/server/core/lib/live/live-utils.ts index 55b7984bf..39051b87a 100644 --- a/server/core/lib/live/live-utils.ts +++ b/server/core/lib/live/live-utils.ts @@ -1,7 +1,7 @@ import { pathExists, remove } from 'fs-extra/esm' import { readdir } from 'fs/promises' import { basename, join } from 'path' -import { LiveVideoLatencyMode, LiveVideoLatencyModeType, VideoStorage } from '@peertube/peertube-models' +import { LiveVideoLatencyMode, LiveVideoLatencyModeType, FileStorage } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' import { VIDEO_LIVE } from '@server/initializers/constants.js' import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js' @@ -24,7 +24,7 @@ async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStre const hlsDirectory = getLiveDirectory(video) // We uploaded files to object storage too, remove them - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) } @@ -86,7 +86,7 @@ async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { } async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { - if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return + if (streamingPlaylist.storage !== FileStorage.OBJECT_STORAGE) return logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.ts index f5343b98f..99ee0584a 100644 --- a/server/core/lib/live/shared/muxing-session.ts +++ b/server/core/lib/live/shared/muxing-session.ts @@ -14,14 +14,14 @@ import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFile import { VideoFileModel } from '@server/models/video/video-file.js' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models/index.js' -import { LiveVideoError, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { LiveVideoError, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths.js' -import { isAbleToUploadVideo } from '../../user.js' +import { isUserQuotaValid } from '../../user.js' import { LiveQuotaStore } from '../live-quota-store.js' import { LiveSegmentShaStore } from '../live-segment-sha-store.js' import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils.js' @@ -95,7 +95,7 @@ class MuxingSession extends EventEmitter { private aborted = false private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { - return isAbleToUploadVideo(userId, 1000) + return isUserQuotaValid({ userId, uploadSize: 1000 }) }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { @@ -186,7 +186,7 @@ class MuxingSession extends EventEmitter { if (this.masterPlaylistCreated === true) return try { - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { let masterContent = await readFile(path, 'utf-8') // If the disk sync is slow, don't upload an empty master playlist on object storage @@ -260,7 +260,7 @@ class MuxingSession extends EventEmitter { logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) } - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { try { await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) } catch (err) { @@ -345,7 +345,7 @@ class MuxingSession extends EventEmitter { await this.addSegmentToReplay(segmentPath) } - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { try { await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) @@ -464,8 +464,8 @@ class MuxingSession extends EventEmitter { playlist.type = VideoStreamingPlaylistType.HLS playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED - ? VideoStorage.OBJECT_STORAGE - : VideoStorage.FILE_SYSTEM + ? FileStorage.OBJECT_STORAGE + : FileStorage.FILE_SYSTEM return playlist.save() } diff --git a/server/core/lib/local-actor.ts b/server/core/lib/local-actor.ts index 5ee9df875..c48127a2c 100644 --- a/server/core/lib/local-actor.ts +++ b/server/core/lib/local-actor.ts @@ -30,13 +30,16 @@ export function buildActorInstance (type: ActivityPubActorType, url: string, pre }) as MActor } -export async function updateLocalActorImageFiles ( - accountOrChannel: MAccountDefault | MChannelDefault, - imagePhysicalFile: Express.Multer.File, +export async function updateLocalActorImageFiles (options: { + accountOrChannel: MAccountDefault | MChannelDefault + imagePhysicalFile: { path: string } type: ActorImageType_Type -) { + sendActorUpdate: boolean +}) { + const { accountOrChannel, imagePhysicalFile, type, sendActorUpdate } = options + const processImageSize = async (imageSize: { width: number, height: number }) => { - const extension = getLowercaseExtension(imagePhysicalFile.filename) + const extension = getLowercaseExtension(imagePhysicalFile.path) const imageName = buildUUID() + extension const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) @@ -63,7 +66,9 @@ export async function updateLocalActorImageFiles ( const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t) await updatedActor.save({ transaction: t }) - await sendUpdateActor(accountOrChannel, t) + if (sendActorUpdate) { + await sendUpdateActor(accountOrChannel, t) + } return type === ActorImageType.AVATAR ? updatedActor.Avatars diff --git a/server/core/lib/model-loaders/video.ts b/server/core/lib/model-loaders/video.ts index 5ea2b67aa..ad8e87155 100644 --- a/server/core/lib/model-loaders/video.ts +++ b/server/core/lib/model-loaders/video.ts @@ -1,3 +1,4 @@ +import { CONFIG } from '@server/initializers/config.js' import { VideoModel } from '@server/models/video/video.js' import { MVideoAccountLightBlacklistAllFiles, @@ -7,6 +8,7 @@ import { MVideoImmutable, MVideoThumbnail } from '@server/types/models/index.js' +import { getOrCreateAPVideo } from '../activitypub/videos/get.js' type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' @@ -50,17 +52,36 @@ function loadVideoByUrl ( url: string, fetchType: VideoLoadByUrlType ): Promise { - if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) + if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccountAndFiles(url) if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) if (fetchType === 'only-video') return VideoModel.loadByUrl(url) } +async function loadOrCreateVideoIfAllowedForUser (videoUrl: string) { + if (CONFIG.SEARCH.REMOTE_URI.USERS) { + try { + const res = await getOrCreateAPVideo({ + videoObject: videoUrl, + fetchType: 'only-immutable-attributes', + allowRefresh: false + }) + + return res?.video + } catch { + return undefined + } + } + + return VideoModel.loadByUrlImmutableAttributes(videoUrl) +} + export { type VideoLoadType, type VideoLoadByUrlType, loadVideo, - loadVideoByUrl + loadVideoByUrl, + loadOrCreateVideoIfAllowedForUser } diff --git a/server/core/lib/moderation.ts b/server/core/lib/moderation.ts index d02a12a30..66ca1d308 100644 --- a/server/core/lib/moderation.ts +++ b/server/core/lib/moderation.ts @@ -17,6 +17,7 @@ import { MCommentAbuseAccountVideo, MCommentOwnerVideo, MUser, + MUserDefault, MVideoAbuseVideoFull, MVideoAccountLightBlacklistAllFiles } from '@server/types/models/index.js' @@ -38,7 +39,7 @@ export type AcceptResult = { function isLocalVideoFileAccepted (object: { videoBody: VideoCreate videoFile: VideoUploadFile - user: UserModel + user: MUserDefault }): AcceptResult { return { accepted: true } } diff --git a/server/core/lib/object-storage/keys.ts b/server/core/lib/object-storage/keys.ts index 9e52a7f19..d5381454c 100644 --- a/server/core/lib/object-storage/keys.ts +++ b/server/core/lib/object-storage/keys.ts @@ -13,8 +13,13 @@ function generateWebVideoObjectStorageKey (filename: string) { return filename } +function generateUserExportObjectStorageKey (filename: string) { + return filename +} + export { generateHLSObjectStorageKey, generateHLSObjectBaseStorageKey, - generateWebVideoObjectStorageKey + generateWebVideoObjectStorageKey, + generateUserExportObjectStorageKey } diff --git a/server/core/lib/object-storage/pre-signed-urls.ts b/server/core/lib/object-storage/pre-signed-urls.ts index 97e219d8e..81d11307b 100644 --- a/server/core/lib/object-storage/pre-signed-urls.ts +++ b/server/core/lib/object-storage/pre-signed-urls.ts @@ -1,6 +1,6 @@ import { CONFIG } from '@server/initializers/config.js' -import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models/index.js' -import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' +import { MStreamingPlaylistVideo, MUserExport, MVideoFile } from '@server/types/models/index.js' +import { generateHLSObjectStorageKey, generateUserExportObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' import { buildKey, getClient } from './shared/index.js' import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js' @@ -10,19 +10,12 @@ export async function generateWebVideoPresignedUrl (options: { }) { const { file, downloadFilename } = options - const key = generateWebVideoObjectStorageKey(file.filename) - - const { GetObjectCommand } = await import('@aws-sdk/client-s3') - const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner') - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), - ResponseContentDisposition: `attachment; filename="${encodeURI(downloadFilename)}"` + const url = await generatePresignedUrl({ + bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, + key: buildKey(generateWebVideoObjectStorageKey(file.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS), + downloadFilename }) - const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 }) - return getWebVideoPublicFileUrl(url) } @@ -33,18 +26,49 @@ export async function generateHLSFilePresignedUrl (options: { }) { const { streamingPlaylist, file, downloadFilename } = options - const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) + const url = await generatePresignedUrl({ + bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, + key: buildKey(generateHLSObjectStorageKey(streamingPlaylist, file.filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), + downloadFilename + }) + + return getHLSPublicFileUrl(url) +} + +export async function generateUserExportPresignedUrl (options: { + userExport: MUserExport + downloadFilename: string +}) { + const { userExport, downloadFilename } = options + + const url = await generatePresignedUrl({ + bucket: CONFIG.OBJECT_STORAGE.USER_EXPORTS.BUCKET_NAME, + key: buildKey(generateUserExportObjectStorageKey(userExport.filename), CONFIG.OBJECT_STORAGE.USER_EXPORTS), + downloadFilename + }) + + return getHLSPublicFileUrl(url) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function generatePresignedUrl (options: { + bucket: string + key: string + downloadFilename: string +}) { + const { bucket, downloadFilename, key } = options const { GetObjectCommand } = await import('@aws-sdk/client-s3') const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner') const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), + Bucket: bucket, + Key: key, ResponseContentDisposition: `attachment; filename="${encodeURI(downloadFilename)}"` }) - const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 }) - - return getHLSPublicFileUrl(url) + return getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 }) } diff --git a/server/core/lib/object-storage/shared/object-storage-helpers.ts b/server/core/lib/object-storage/shared/object-storage-helpers.ts index ce96696e5..a758d204e 100644 --- a/server/core/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/core/lib/object-storage/shared/object-storage-helpers.ts @@ -3,7 +3,7 @@ import { isArray } from '@server/helpers/custom-validators/misc.js' import { logger } from '@server/helpers/logger.js' import { CONFIG } from '@server/initializers/config.js' import Bluebird from 'bluebird' -import { createReadStream, createWriteStream, ReadStream } from 'fs' +import { createReadStream, createWriteStream } from 'fs' import { ensureDir } from 'fs-extra/esm' import { dirname } from 'path' import { Readable } from 'stream' @@ -67,6 +67,19 @@ async function storeContent (options: { return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) } +async function storeStream (options: { + stream: Readable + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}): Promise { + const { stream, objectStorageKey, bucketInfo, isPrivate } = options + + logger.debug('Streaming file to %s%s in bucket %s', bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) + + return uploadToStorage({ objectStorageKey, content: stream, bucketInfo, isPrivate }) +} + // --------------------------------------------------------------------------- async function updateObjectACL (options: { @@ -225,6 +238,27 @@ async function createObjectReadStream (options: { // --------------------------------------------------------------------------- +async function getObjectStorageFileSize (options: { + key: string + bucketInfo: BucketInfo +}) { + const { key, bucketInfo } = options + + const { HeadObjectCommand } = await import('@aws-sdk/client-s3') + + const command = new HeadObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: buildKey(key, bucketInfo) + }) + + const client = await getClient() + const response = await client.send(command) + + return response.ContentLength +} + +// --------------------------------------------------------------------------- + export { type BucketInfo, @@ -232,6 +266,7 @@ export { storeObject, storeContent, + storeStream, removeObject, removeObjectByFullKey, @@ -243,13 +278,15 @@ export { updatePrefixACL, listKeysOfPrefix, - createObjectReadStream + createObjectReadStream, + + getObjectStorageFileSize } // --------------------------------------------------------------------------- async function uploadToStorage (options: { - content: ReadStream | string + content: Readable | string objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean diff --git a/server/core/lib/object-storage/user-export.ts b/server/core/lib/object-storage/user-export.ts new file mode 100644 index 000000000..d6ca34ec8 --- /dev/null +++ b/server/core/lib/object-storage/user-export.ts @@ -0,0 +1,25 @@ +import { CONFIG } from '@server/initializers/config.js' +import { MUserExport } from '@server/types/models/index.js' +import { generateUserExportObjectStorageKey } from './keys.js' +import { getObjectStorageFileSize, removeObject, storeStream } from './shared/index.js' +import { Readable } from 'stream' + +export function storeUserExportFile (stream: Readable, userExport: MUserExport) { + return storeStream({ + stream, + objectStorageKey: generateUserExportObjectStorageKey(userExport.filename), + bucketInfo: CONFIG.OBJECT_STORAGE.USER_EXPORTS, + isPrivate: true + }) +} + +export function removeUserExportObjectStorage (userExport: MUserExport) { + return removeObject(generateUserExportObjectStorageKey(userExport.filename), CONFIG.OBJECT_STORAGE.USER_EXPORTS) +} + +export function getUserExportFileObjectStorageSize (userExport: MUserExport) { + return getObjectStorageFileSize({ + key: generateUserExportObjectStorageKey(userExport.filename), + bucketInfo: CONFIG.OBJECT_STORAGE.USER_EXPORTS + }) +} diff --git a/server/core/lib/paths.ts b/server/core/lib/paths.ts index 208e78efa..b9909d4ff 100644 --- a/server/core/lib/paths.ts +++ b/server/core/lib/paths.ts @@ -1,32 +1,40 @@ import { join } from 'path' import { CONFIG } from '@server/initializers/config.js' import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants.js' -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models/index.js' +import { + isStreamingPlaylist, + MStreamingPlaylistVideo, + MUserExport, + MUserImport, + MVideo, + MVideoFile, + MVideoUUID +} from '@server/types/models/index.js' import { removeFragmentedMP4Ext } from '@peertube/peertube-core-utils' import { buildUUID } from '@peertube/peertube-node-utils' import { isVideoInPrivateDirectory } from './video-privacy.js' // ################## Video file name ################## -function generateWebVideoFilename (resolution: number, extname: string) { +export function generateWebVideoFilename (resolution: number, extname: string) { return buildUUID() + '-' + resolution + extname } -function generateHLSVideoFilename (resolution: number) { +export function generateHLSVideoFilename (resolution: number) { return `${buildUUID()}-${resolution}-fragmented.mp4` } // ################## Streaming playlist ################## -function getLiveDirectory (video: MVideo) { +export function getLiveDirectory (video: MVideo) { return getHLSDirectory(video) } -function getLiveReplayBaseDirectory (video: MVideo) { +export function getLiveReplayBaseDirectory (video: MVideo) { return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) } -function getHLSDirectory (video: MVideo) { +export function getHLSDirectory (video: MVideo) { if (isVideoInPrivateDirectory(video.privacy)) { return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) } @@ -34,22 +42,22 @@ function getHLSDirectory (video: MVideo) { return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) } -function getHLSRedundancyDirectory (video: MVideoUUID) { +export function getHLSRedundancyDirectory (video: MVideoUUID) { return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) } -function getHlsResolutionPlaylistFilename (videoFilename: string) { +export function getHlsResolutionPlaylistFilename (videoFilename: string) { // Video file name already contain resolution return removeFragmentedMP4Ext(videoFilename) + '.m3u8' } -function generateHLSMasterPlaylistFilename (isLive = false) { +export function generateHLSMasterPlaylistFilename (isLive = false) { if (isLive) return 'master.m3u8' return buildUUID() + '-master.m3u8' } -function generateHlsSha256SegmentsFilename (isLive = false) { +export function generateHlsSha256SegmentsFilename (isLive = false) { if (isLive) return 'segments-sha256.json' return buildUUID() + '-segments-sha256.json' @@ -57,7 +65,7 @@ function generateHlsSha256SegmentsFilename (isLive = false) { // ################## Torrents ################## -function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { +export function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { const extension = '.torrent' const uuid = buildUUID() @@ -68,25 +76,16 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi return uuid + '-' + resolution + extension } -function getFSTorrentFilePath (videoFile: MVideoFile) { +export function getFSTorrentFilePath (videoFile: MVideoFile) { return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) } // --------------------------------------------------------------------------- -export { - generateHLSVideoFilename, - generateWebVideoFilename, - - generateTorrentFileName, - getFSTorrentFilePath, - - getHLSDirectory, - getLiveDirectory, - getLiveReplayBaseDirectory, - getHLSRedundancyDirectory, - - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getHlsResolutionPlaylistFilename +export function getFSUserExportFilePath (userExport: MUserExport) { + return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, userExport.filename) +} + +export function getFSUserImportFilePath (userImport: MUserImport) { + return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, userImport.filename) } diff --git a/server/core/lib/plugins/plugin-helpers-builder.ts b/server/core/lib/plugins/plugin-helpers-builder.ts index 4c1bd2bf7..506fbabb6 100644 --- a/server/core/lib/plugins/plugin-helpers-builder.ts +++ b/server/core/lib/plugins/plugin-helpers-builder.ts @@ -16,7 +16,7 @@ import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js' import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models/index.js' import { PeerTubeHelpers } from '@server/types/plugins/index.js' import { ffprobePromise } from '@peertube/peertube-ffmpeg' -import { VideoBlacklistCreate, VideoStorage } from '@peertube/peertube-models' +import { VideoBlacklistCreate, FileStorage } from '@peertube/peertube-models' import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist.js' import { PeerTubeSocket } from '../peertube-socket.js' import { ServerConfigManager } from '../server-config-manager.js' @@ -105,7 +105,7 @@ function buildVideosHelpers () { if (!video) return undefined const webVideoFiles = (video.VideoFiles || []).map(f => ({ - path: f.storage === VideoStorage.FILE_SYSTEM + path: f.storage === FileStorage.FILE_SYSTEM ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) : null, url: f.getFileUrl(video), @@ -120,7 +120,7 @@ function buildVideosHelpers () { const hlsVideoFiles = hls ? (video.getHLSPlaylist().VideoFiles || []).map(f => { return { - path: f.storage === VideoStorage.FILE_SYSTEM + path: f.storage === FileStorage.FILE_SYSTEM ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f) : null, url: f.getFileUrl(video), @@ -160,8 +160,13 @@ function buildModerationHelpers () { return { blockServer: async (options: { byAccountId: number, hostToBlock: string }) => { const serverToBlock = await ServerModel.loadOrCreateByHost(options.hostToBlock) + const user = await UserModel.loadByAccountId(options.byAccountId) - await addServerInBlocklist(options.byAccountId, serverToBlock.id) + await addServerInBlocklist({ + byAccountId: options.byAccountId, + targetServerId: serverToBlock.id, + removeNotificationOfUserId: user?.id + }) }, unblockServer: async (options: { byAccountId: number, hostToUnblock: string }) => { @@ -175,7 +180,13 @@ function buildModerationHelpers () { const accountToBlock = await AccountModel.loadByNameWithHost(options.handleToBlock) if (!accountToBlock) return - await addAccountInBlocklist(options.byAccountId, accountToBlock.id) + const user = await UserModel.loadByAccountId(options.byAccountId) + + await addAccountInBlocklist({ + byAccountId: options.byAccountId, + targetAccountId: accountToBlock.id, + removeNotificationOfUserId: user?.id + }) }, unblockAccount: async (options: { byAccountId: number, handleToUnblock: string }) => { diff --git a/server/core/lib/rate.ts b/server/core/lib/rate.ts new file mode 100644 index 000000000..1e047da5a --- /dev/null +++ b/server/core/lib/rate.ts @@ -0,0 +1,64 @@ +import { VIDEO_RATE_TYPES } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js' +import { AccountModel } from '@server/models/account/account.js' +import { getLocalRateUrl, sendVideoRateChange } from './activitypub/video-rates.js' +import { MAccountId, MAccountUrl, MVideoFullLight } from '@server/types/models/index.js' +import { UserVideoRateType } from '@peertube/peertube-models' + +export async function userRateVideo (options: { + rateType: UserVideoRateType + account: MAccountUrl & MAccountId + video: MVideoFullLight +}) { + const { account, rateType, video } = options + + await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const accountInstance = await AccountModel.load(account.id, t) + const previousRate = await AccountVideoRateModel.load(accountInstance.id, video.id, t) + + // Same rate, nothing do to + if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return + + let likesToIncrement = 0 + let dislikesToIncrement = 0 + + if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++ + else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ + + // There was a previous rate, update it + if (previousRate) { + // We will remove the previous rate, so we will need to update the video count attribute + if (previousRate.type === 'like') likesToIncrement-- + else if (previousRate.type === 'dislike') dislikesToIncrement-- + + if (rateType === 'none') { // Destroy previous rate + await previousRate.destroy(sequelizeOptions) + } else { // Update previous rate + previousRate.type = rateType + previousRate.url = getLocalRateUrl(rateType, account.Actor, video) + await previousRate.save(sequelizeOptions) + } + } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate + const query = { + accountId: accountInstance.id, + videoId: video.id, + type: rateType, + url: getLocalRateUrl(rateType, account.Actor, video) + } + + await AccountVideoRateModel.create(query, sequelizeOptions) + } + + const incrementQuery = { + likes: likesToIncrement, + dislikes: dislikesToIncrement + } + + await video.increment(incrementQuery, sequelizeOptions) + + await sendVideoRateChange(accountInstance, video, likesToIncrement, dislikesToIncrement, t) + }) +} diff --git a/server/core/lib/schedulers/remove-expired-user-exports-scheduler.ts b/server/core/lib/schedulers/remove-expired-user-exports-scheduler.ts new file mode 100644 index 000000000..bf3a752f2 --- /dev/null +++ b/server/core/lib/schedulers/remove-expired-user-exports-scheduler.ts @@ -0,0 +1,30 @@ +import { logger } from '../../helpers/logger.js' +import { AbstractScheduler } from './abstract-scheduler.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { CONFIG } from '../../initializers/config.js' +import { UserExportModel } from '@server/models/user/user-export.js' + +export class RemoveExpiredUserExportsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_EXPIRED_USER_EXPORTS + + private constructor () { + super() + } + + protected async internalExecute () { + const expired = await UserExportModel.listExpired(CONFIG.EXPORT.USERS.EXPORT_EXPIRATION) + + for (const userExport of expired) { + logger.info('Removing expired user exports ' + userExport.filename) + + await userExport.destroy() + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/core/lib/schedulers/update-videos-scheduler.ts b/server/core/lib/schedulers/update-videos-scheduler.ts index 50ecb019c..d2cebdbd4 100644 --- a/server/core/lib/schedulers/update-videos-scheduler.ts +++ b/server/core/lib/schedulers/update-videos-scheduler.ts @@ -8,7 +8,7 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda import { Notifier } from '../notifier/index.js' import { VideoPathManager } from '../video-path-manager.js' import { setVideoPrivacy } from '../video-privacy.js' -import { addVideoJobsAfterUpdate } from '../video.js' +import { addVideoJobsAfterUpdate } from '../video-jobs.js' import { AbstractScheduler } from './abstract-scheduler.js' export class UpdateVideosScheduler extends AbstractScheduler { diff --git a/server/core/lib/server-config-manager.ts b/server/core/lib/server-config-manager.ts index 3b874edfd..977ede1fc 100644 --- a/server/core/lib/server-config-manager.ts +++ b/server/core/lib/server-config-manager.ts @@ -193,6 +193,16 @@ class ServerConfigManager { }, videoChannelSynchronization: { enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED + }, + users: { + enabled: CONFIG.IMPORT.USERS.ENABLED + } + }, + export: { + users: { + enabled: CONFIG.EXPORT.USERS.ENABLED, + exportExpiration: CONFIG.EXPORT.USERS.EXPORT_EXPIRATION, + maxUserVideoQuota: CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA } }, autoBlacklist: { diff --git a/server/core/lib/thumbnail.ts b/server/core/lib/thumbnail.ts index 1362f9e4d..14b849fb4 100644 --- a/server/core/lib/thumbnail.ts +++ b/server/core/lib/thumbnail.ts @@ -11,7 +11,7 @@ import { VideoPathManager } from './video-path-manager.js' import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process.js' import { generateThumbnailFromVideo } from '@server/helpers/ffmpeg/ffmpeg-image.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' -import { remove } from 'fs-extra' +import { remove } from 'fs-extra/esm' import { FfprobeData } from 'fluent-ffmpeg' import Bluebird from 'bluebird' diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.ts index a5b934048..8e07a5f37 100644 --- a/server/core/lib/transcoding/web-transcoding.ts +++ b/server/core/lib/transcoding/web-transcoding.ts @@ -2,7 +2,7 @@ import { Job } from 'bullmq' import { move, remove } from 'fs-extra/esm' import { copyFile, stat } from 'fs/promises' import { basename, join } from 'path' -import { VideoStorage } from '@peertube/peertube-models' +import { FileStorage } from '@peertube/peertube-models' import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' import { VideoModel } from '@server/models/video/video.js' @@ -16,7 +16,7 @@ import { buildFileMetadata } from '../video-file.js' import { VideoPathManager } from '../video-path-manager.js' import { buildFFmpegVOD } from './shared/index.js' import { buildOriginalFileResolution } from './transcoding-resolutions.js' -import { buildStoryboardJobIfNeeded } from '../video.js' +import { buildStoryboardJobIfNeeded } from '../video-jobs.js' // Optimize the original video file and replace it. The resolution is not changed. export async function optimizeOriginalVideofile (options: { @@ -66,7 +66,7 @@ export async function optimizeOriginalVideofile (options: { inputVideoFile.resolution = resolution inputVideoFile.extname = newExtname inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) - inputVideoFile.storage = VideoStorage.FILE_SYSTEM + inputVideoFile.storage = FileStorage.FILE_SYSTEM const { videoFile } = await onWebVideoFileTranscoding({ video, diff --git a/server/core/lib/user-import-export/exporters/abstract-user-exporter.ts b/server/core/lib/user-import-export/exporters/abstract-user-exporter.ts new file mode 100644 index 000000000..8909a66ae --- /dev/null +++ b/server/core/lib/user-import-export/exporters/abstract-user-exporter.ts @@ -0,0 +1,51 @@ +import { Activity, ActivityPubActor, ActivityPubOrderedCollection } from '@peertube/peertube-models' +import { Awaitable } from '@peertube/peertube-typescript-utils' +import { MUserDefault } from '@server/types/models/user/user.js' +import { Readable } from 'stream' + +export type ExportResult = { + json: T[] | T + + staticFiles: { + archivePath: string + createrReadStream: () => Promise + }[] + + activityPub?: ActivityPubActor | ActivityPubOrderedCollection + + activityPubOutbox?: Omit[] +} + +type ActivityPubFilenames = { + likes: string + dislikes: string + outbox: string + following: string + account: string +} + +export abstract class AbstractUserExporter { + protected user: MUserDefault + + protected activityPubFilenames: ActivityPubFilenames + + protected relativeStaticDirPath: string + + constructor (options: { + user: MUserDefault + + activityPubFilenames: ActivityPubFilenames + + relativeStaticDirPath?: string + }) { + this.user = options.user + this.activityPubFilenames = options.activityPubFilenames + this.relativeStaticDirPath = options.relativeStaticDirPath + } + + getActivityPubFilename () { + return null + } + + abstract export (): Awaitable> +} diff --git a/server/core/lib/user-import-export/exporters/account-exporter.ts b/server/core/lib/user-import-export/exporters/account-exporter.ts new file mode 100644 index 000000000..4fe3cc96b --- /dev/null +++ b/server/core/lib/user-import-export/exporters/account-exporter.ts @@ -0,0 +1,68 @@ +import { AccountExportJSON, ActivityPubActor, ActorImageType } from '@peertube/peertube-models' +import { MAccountDefault, MActorDefaultBanner } from '@server/types/models/index.js' +import { ActorExporter } from './actor-exporter.js' +import { AccountModel } from '@server/models/account/account.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { join } from 'path' + +export class AccountExporter extends ActorExporter { + + async export () { + const account = await AccountModel.loadLocalByName(this.user.username) + + const { staticFiles, relativePathsFromJSON } = this.exportActorFiles(account.Actor as MActorDefaultBanner) + + return { + json: this.exportAccountJSON(account, relativePathsFromJSON), + staticFiles, + activityPub: await this.exportAccountAP(account) + } + } + + getActivityPubFilename () { + return this.activityPubFilenames.account + } + + // --------------------------------------------------------------------------- + + private exportAccountJSON (account: MAccountDefault, archiveFiles: { avatar: string }): AccountExportJSON { + return { + ...this.exportActorJSON(account.Actor as MActorDefaultBanner), + + displayName: account.getDisplayName(), + description: account.description, + + updatedAt: account.updatedAt.toISOString(), + createdAt: account.createdAt.toISOString(), + + archiveFiles + } + } + + private async exportAccountAP (account: MAccountDefault): Promise { + const avatar = account.Actor.getMaxQualityImage(ActorImageType.AVATAR) + + return activityPubContextify( + { + ...await account.toActivityPubObject(), + + likes: this.activityPubFilenames.likes, + dislikes: this.activityPubFilenames.dislikes, + outbox: this.activityPubFilenames.outbox, + following: this.activityPubFilenames.following, + + icon: avatar + ? [ + { + ...avatar.toActivityPubObject(), + + url: join(this.relativeStaticDirPath, this.getAvatarPath(account.Actor, avatar.filename)) + } + ] + : [] + }, + 'Actor', + getContextFilter()) + } +} diff --git a/server/core/lib/user-import-export/exporters/actor-exporter.ts b/server/core/lib/user-import-export/exporters/actor-exporter.ts new file mode 100644 index 000000000..8b48d08e7 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/actor-exporter.ts @@ -0,0 +1,78 @@ +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js' +import { ActorImageType } from '@peertube/peertube-models' +import { MActor, MActorDefaultBanner, MActorImage } from '@server/types/models/index.js' +import { extname, join } from 'path' +import { createReadStream } from 'fs' + +export abstract class ActorExporter extends AbstractUserExporter { + + protected exportActorJSON (actor: MActorDefaultBanner) { + return { + url: actor.url, + + name: actor.preferredUsername, + + avatars: this.exportActorImageJSON(actor.Avatars), + banners: actor.hasImage(ActorImageType.BANNER) + ? this.exportActorImageJSON(actor.Banners) + : [] + } + } + + protected exportActorImageJSON (images: MActorImage[]) { + return images.map(i => ({ + width: i.width, + url: ActorImageModel.getImageUrl(i), + createdAt: i.createdAt.toISOString(), + updatedAt: i.updatedAt.toISOString() + })) + } + + // --------------------------------------------------------------------------- + + protected exportActorFiles (actor: MActorDefaultBanner) { + const staticFiles: ExportResult['staticFiles'] = [] + const relativePathsFromJSON = { + avatar: null as string, + banner: null as string + } + + const toProcess = [ + { + archivePathBuilder: (filename: string) => this.getBannerPath(actor, filename), + type: ActorImageType.BANNER + }, + { + archivePathBuilder: (filename: string) => this.getAvatarPath(actor, filename), + type: ActorImageType.AVATAR + } + ] + + for (const { archivePathBuilder, type } of toProcess) { + if (!actor.hasImage(type)) continue + + const image = actor.getMaxQualityImage(type) + + staticFiles.push({ + archivePath: archivePathBuilder(image.filename), + createrReadStream: () => Promise.resolve(createReadStream(image.getPath())) + }) + + const relativePath = join(this.relativeStaticDirPath, archivePathBuilder(image.filename)) + + if (type === ActorImageType.AVATAR) relativePathsFromJSON.avatar = relativePath + else if (type === ActorImageType.BANNER) relativePathsFromJSON.banner = relativePath + } + + return { staticFiles, relativePathsFromJSON } + } + + protected getAvatarPath (actor: MActor, filename: string) { + return join('avatars', actor.preferredUsername + extname(filename)) + } + + protected getBannerPath (actor: MActor, filename: string) { + return join('banners', actor.preferredUsername + extname(filename)) + } +} diff --git a/server/core/lib/user-import-export/exporters/blocklist-exporter.ts b/server/core/lib/user-import-export/exporters/blocklist-exporter.ts new file mode 100644 index 000000000..74b3a5b37 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/blocklist-exporter.ts @@ -0,0 +1,24 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { BlocklistExportJSON } from '@peertube/peertube-models' + +export class BlocklistExporter extends AbstractUserExporter { + + async export () { + const [ instancesBlocklist, accountsBlocklist ] = await Promise.all([ + ServerBlocklistModel.listHostsBlockedBy([ this.user.Account.id ]), + AccountBlocklistModel.listHandlesBlockedBy([ this.user.Account.id ]) + ]) + + return { + json: { + instances: instancesBlocklist.map(b => ({ host: b })), + actors: accountsBlocklist.map(h => ({ handle: h })) + } as BlocklistExportJSON, + + staticFiles: [] + } + } + +} diff --git a/server/core/lib/user-import-export/exporters/channels-exporter.ts b/server/core/lib/user-import-export/exporters/channels-exporter.ts new file mode 100644 index 000000000..0cfd7bc1a --- /dev/null +++ b/server/core/lib/user-import-export/exporters/channels-exporter.ts @@ -0,0 +1,64 @@ +import { logger } from '@server/helpers/logger.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { ExportResult } from './abstract-user-exporter.js' +import { ChannelExportJSON } from '@peertube/peertube-models' +import { MChannelBannerAccountDefault } from '@server/types/models/index.js' +import { ActorExporter } from './actor-exporter.js' + +export class ChannelsExporter extends ActorExporter { + + async export () { + const channelsJSON: ChannelExportJSON['channels'] = [] + let staticFiles: ExportResult['staticFiles'] = [] + + const channels = await VideoChannelModel.listAllByAccount(this.user.Account.id) + + for (const channel of channels) { + try { + const exported = await this.exportChannel(channel.id) + + channelsJSON.push(exported.json) + staticFiles = staticFiles.concat(exported.staticFiles) + } catch (err) { + logger.warn('Cannot export channel %s.', channel.name, { err }) + } + } + + return { + json: { channels: channelsJSON }, + staticFiles + } + } + + private async exportChannel (channelId: number) { + const channel = await VideoChannelModel.loadAndPopulateAccount(channelId) + + const { relativePathsFromJSON, staticFiles } = this.exportActorFiles(channel.Actor) + + return { + json: this.exportChannelJSON(channel, relativePathsFromJSON), + staticFiles + } + } + + // --------------------------------------------------------------------------- + + private exportChannelJSON ( + channel: MChannelBannerAccountDefault, + archiveFiles: { avatar: string, banner: string } + ): ChannelExportJSON['channels'][0] { + return { + ...this.exportActorJSON(channel.Actor), + + displayName: channel.getDisplayName(), + description: channel.description, + support: channel.support, + + updatedAt: channel.updatedAt.toISOString(), + createdAt: channel.createdAt.toISOString(), + + archiveFiles + } + } + +} diff --git a/server/core/lib/user-import-export/exporters/comments-exporter.ts b/server/core/lib/user-import-export/exporters/comments-exporter.ts new file mode 100644 index 000000000..47f850118 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/comments-exporter.ts @@ -0,0 +1,50 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { MCommentExport } from '@server/types/models/index.js' +import { CommentsExportJSON, VideoCommentObject } from '@peertube/peertube-models' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import Bluebird from 'bluebird' +import { audiencify, getAudience } from '@server/lib/activitypub/audience.js' +import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js' + +export class CommentsExporter extends AbstractUserExporter { + + async export () { + const comments = await VideoCommentModel.listForExport(this.user.Account.id) + + return { + json: { + comments: this.formatCommentsJSON(comments) + }, + + activityPubOutbox: await this.formatCommentsAP(comments), + + staticFiles: [] + } + } + + private formatCommentsJSON (comments: MCommentExport[]) { + return comments.map(c => ({ + url: c.url, + text: c.text, + createdAt: c.createdAt.toISOString(), + videoUrl: c.Video.url, + inReplyToCommentUrl: c?.InReplyToVideoComment?.url + })) + } + + private formatCommentsAP (comments: MCommentExport[]) { + return Bluebird.mapSeries(comments, async ({ url }) => { + const comment = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) + + const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, undefined) + let commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject + + const isPublic = true // Comments are always public + const audience = getAudience(comment.Account.Actor, isPublic) + + commentObject = audiencify(commentObject, audience) + + return buildCreateActivity(comment.url, comment.Account.Actor, commentObject, audience) + }) + } +} diff --git a/server/core/lib/user-import-export/exporters/dislikes-exporter.ts b/server/core/lib/user-import-export/exporters/dislikes-exporter.ts new file mode 100644 index 000000000..0faf47ac4 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/dislikes-exporter.ts @@ -0,0 +1,41 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js' +import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js' +import { ActivityPubOrderedCollection, DislikesExportJSON } from '@peertube/peertube-models' +import { activityPubCollection } from '@server/lib/activitypub/collection.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' + +export class DislikesExporter extends AbstractUserExporter { + + async export () { + const dislikes = await AccountVideoRateModel.listRatesOfAccountId(this.user.Account.id, 'dislike') + + return { + json: { + dislikes: this.formatDislikesJSON(dislikes) + } as DislikesExportJSON, + + activityPub: await this.formatDislikesAP(dislikes), + + staticFiles: [] + } + } + + getActivityPubFilename () { + return this.activityPubFilenames.dislikes + } + + private formatDislikesJSON (dislikes: MAccountVideoRateVideoUrl[]) { + return dislikes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() })) + } + + private formatDislikesAP (dislikes: MAccountVideoRateVideoUrl[]): Promise> { + return activityPubContextify( + activityPubCollection(this.getActivityPubFilename(), dislikes.map(l => l.Video.url)), + 'Rate', + getContextFilter() + ) + } + +} diff --git a/server/core/lib/user-import-export/exporters/followers-exporter.ts b/server/core/lib/user-import-export/exporters/followers-exporter.ts new file mode 100644 index 000000000..0692ed4f2 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/followers-exporter.ts @@ -0,0 +1,45 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { FollowersExportJSON } from '@peertube/peertube-models' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' + +export class FollowersExporter extends AbstractUserExporter { + + async export () { + let followersJSON = this.formatFollowersJSON( + await ActorFollowModel.listAcceptedFollowersForExport(this.user.Account.actorId), + this.user.Account.Actor.getFullIdentifier() + ) + + const channels = await VideoChannelModel.listAllByAccount(this.user.Account.id) + + for (const channel of channels) { + followersJSON = followersJSON.concat( + this.formatFollowersJSON( + await ActorFollowModel.listAcceptedFollowersForExport(channel.Actor.id), + channel.Actor.getFullIdentifier() + ) + ) + } + + return { + json: { followers: followersJSON } as FollowersExportJSON, + + staticFiles: [] + } + } + + private formatFollowersJSON ( + follows: { + createdAt: Date + followerHandle: string + }[], + targetHandle: string + ): FollowersExportJSON['followers'] { + return follows.map(f => ({ + targetHandle, + handle: f.followerHandle, + createdAt: f.createdAt.toISOString() + })) + } +} diff --git a/server/core/lib/user-import-export/exporters/following-exporter.ts b/server/core/lib/user-import-export/exporters/following-exporter.ts new file mode 100644 index 000000000..0a04e2f74 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/following-exporter.ts @@ -0,0 +1,49 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { ActivityPubOrderedCollection, FollowingExportJSON } from '@peertube/peertube-models' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { activityPubCollection } from '@server/lib/activitypub/collection.js' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' + +export class FollowingExporter extends AbstractUserExporter { + + async export () { + const following = await ActorFollowModel.listAcceptedFollowingForExport(this.user.Account.actorId) + const followingJSON = this.formatFollowingJSON(following, this.user.Account.Actor.getFullIdentifier()) + + return { + json: { following: followingJSON } as FollowingExportJSON, + + staticFiles: [], + + activityPub: await this.formatFollowingAP(following) + } + } + + getActivityPubFilename () { + return this.activityPubFilenames.following + } + + private formatFollowingJSON ( + follows: { + createdAt: Date + followingHandle: string + }[], + handle: string + ): FollowingExportJSON['following'] { + return follows.map(f => ({ + handle, + targetHandle: f.followingHandle, + createdAt: f.createdAt.toISOString() + })) + } + + private formatFollowingAP (follows: { followingUrl: string }[]): Promise> { + return activityPubContextify( + activityPubCollection(this.getActivityPubFilename(), follows.map(f => f.followingUrl)), + 'Collection', + getContextFilter() + ) + } + +} diff --git a/server/core/lib/user-import-export/exporters/index.ts b/server/core/lib/user-import-export/exporters/index.ts new file mode 100644 index 000000000..eb4fce840 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/index.ts @@ -0,0 +1,12 @@ +export * from './account-exporter.js' +export * from './blocklist-exporter.js' +export * from './channels-exporter.js' +export * from './comments-exporter.js' +export * from './dislikes-exporter.js' +export * from './followers-exporter.js' +export * from './following-exporter.js' +export * from './likes-exporter.js' +export * from './abstract-user-exporter.js' +export * from './user-settings-exporter.js' +export * from './video-playlists-exporter.js' +export * from './videos-exporter.js' diff --git a/server/core/lib/user-import-export/exporters/likes-exporter.ts b/server/core/lib/user-import-export/exporters/likes-exporter.ts new file mode 100644 index 000000000..9f54b7030 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/likes-exporter.ts @@ -0,0 +1,40 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { MAccountVideoRateVideoUrl } from '@server/types/models/index.js' +import { ActivityPubOrderedCollection, LikesExportJSON } from '@peertube/peertube-models' +import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js' +import { activityPubCollection } from '@server/lib/activitypub/collection.js' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' + +export class LikesExporter extends AbstractUserExporter { + + async export () { + const likes = await AccountVideoRateModel.listRatesOfAccountId(this.user.Account.id, 'like') + + return { + json: { + likes: this.formatLikesJSON(likes) + } as LikesExportJSON, + + activityPub: await this.formatLikesAP(likes), + + staticFiles: [] + } + } + + getActivityPubFilename () { + return this.activityPubFilenames.likes + } + + private formatLikesJSON (likes: MAccountVideoRateVideoUrl[]) { + return likes.map(o => ({ videoUrl: o.Video.url, createdAt: o.createdAt.toISOString() })) + } + + private formatLikesAP (likes: MAccountVideoRateVideoUrl[]): Promise> { + return activityPubContextify( + activityPubCollection(this.getActivityPubFilename(), likes.map(l => l.Video.url)), + 'Collection', + getContextFilter() + ) + } +} diff --git a/server/core/lib/user-import-export/exporters/user-settings-exporter.ts b/server/core/lib/user-import-export/exporters/user-settings-exporter.ts new file mode 100644 index 000000000..f63590493 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/user-settings-exporter.ts @@ -0,0 +1,33 @@ +import { AbstractUserExporter } from './abstract-user-exporter.js' +import { UserSettingsExportJSON } from '@peertube/peertube-models' + +export class UserSettingsExporter extends AbstractUserExporter { + + export () { + return { + json: { + email: this.user.email, + + emailPublic: this.user.emailPublic, + nsfwPolicy: this.user.nsfwPolicy, + + autoPlayVideo: this.user.autoPlayVideo, + autoPlayNextVideo: this.user.autoPlayNextVideo, + autoPlayNextVideoPlaylist: this.user.autoPlayNextVideoPlaylist, + + p2pEnabled: this.user.p2pEnabled, + + videosHistoryEnabled: this.user.videosHistoryEnabled, + videoLanguages: this.user.videoLanguages, + + theme: this.user.theme, + + createdAt: this.user.createdAt, + + notificationSettings: this.user.NotificationSetting.toFormattedJSON() + }, + + staticFiles: [] + } + } +} diff --git a/server/core/lib/user-import-export/exporters/video-playlists-exporter.ts b/server/core/lib/user-import-export/exporters/video-playlists-exporter.ts new file mode 100644 index 000000000..0645cf244 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/video-playlists-exporter.ts @@ -0,0 +1,75 @@ +import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js' +import { VideoPlaylistsExportJSON } from '@peertube/peertube-models' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js' +import { extname, join } from 'path' +import { createReadStream } from 'fs' +import { MThumbnail, MVideoPlaylist } from '@server/types/models/index.js' + +export class VideoPlaylistsExporter extends AbstractUserExporter { + + async export () { + const playlistsJSON: VideoPlaylistsExportJSON['videoPlaylists'] = [] + const staticFiles: ExportResult['staticFiles'] = [] + + const playlists = await VideoPlaylistModel.listPlaylistForExport(this.user.Account.id) + + for (const playlist of playlists) { + const elements = await VideoPlaylistElementModel.listElementsForExport(playlist.id) + + const archiveFiles = { + thumbnail: null as string + } + + if (playlist.hasThumbnail()) { + const thumbnail = playlist.Thumbnail + + staticFiles.push({ + archivePath: this.getArchiveThumbnailPath(playlist, thumbnail), + createrReadStream: () => Promise.resolve(createReadStream(thumbnail.getPath())) + }) + + archiveFiles.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailPath(playlist, thumbnail)) + } + + playlistsJSON.push({ + displayName: playlist.name, + description: playlist.description, + privacy: playlist.privacy, + url: playlist.url, + uuid: playlist.uuid, + + type: playlist.type, + + channel: { + name: playlist.VideoChannel?.Actor?.preferredUsername + }, + + createdAt: playlist.createdAt.toISOString(), + updatedAt: playlist.updatedAt.toISOString(), + + thumbnailUrl: playlist.Thumbnail?.getOriginFileUrl(playlist), + + elements: elements.map(e => ({ + videoUrl: e.Video.url, + startTimestamp: e.startTimestamp, + stopTimestamp: e.stopTimestamp + })), + + archiveFiles + }) + } + + return { + json: { + videoPlaylists: playlistsJSON + }, + + staticFiles + } + } + + private getArchiveThumbnailPath (playlist: MVideoPlaylist, thumbnail: MThumbnail) { + return join('thumbnails', playlist.uuid + extname(thumbnail.filename)) + } +} diff --git a/server/core/lib/user-import-export/exporters/videos-exporter.ts b/server/core/lib/user-import-export/exporters/videos-exporter.ts new file mode 100644 index 000000000..8110cc097 --- /dev/null +++ b/server/core/lib/user-import-export/exporters/videos-exporter.ts @@ -0,0 +1,329 @@ +import { VideoModel } from '@server/models/video/video.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js' +import { + MStreamingPlaylistFiles, + MThumbnail, MVideo, MVideoAP, MVideoCaption, + MVideoCaptionLanguageUrl, + MVideoFile, + MVideoFullLight, MVideoLiveWithSetting, + MVideoPassword +} from '@server/types/models/index.js' +import { logger } from '@server/helpers/logger.js' +import { ActivityCreate, VideoExportJSON, VideoObject, VideoPrivacy, FileStorage } from '@peertube/peertube-models' +import Bluebird from 'bluebird' +import { getHLSFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js' +import { createReadStream } from 'fs' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { extname, join } from 'path' +import { Readable } from 'stream' +import { getAudience, audiencify } from '@server/lib/activitypub/audience.js' +import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js' +import { pick } from '@peertube/peertube-core-utils' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' + +export class VideosExporter extends AbstractUserExporter { + + constructor (private readonly options: ConstructorParameters>[0] & { + withVideoFiles: boolean + }) { + super(options) + } + + async export () { + const videosJSON: VideoExportJSON['videos'] = [] + const activityPubOutbox: ActivityCreate[] = [] + let staticFiles: ExportResult['staticFiles'] = [] + + const channels = await VideoChannelModel.listAllByAccount(this.user.Account.id) + + for (const channel of channels) { + const videoIds = await VideoModel.getAllIdsFromChannel(channel) + + await Bluebird.map(videoIds, async id => { + try { + const exported = await this.exportVideo(id) + + videosJSON.push(exported.json) + staticFiles = staticFiles.concat(exported.staticFiles) + activityPubOutbox.push(exported.activityPubOutbox) + } catch (err) { + logger.warn('Cannot export video %d.', id, { err }) + } + }, { concurrency: 10 }) + } + + return { + json: { videos: videosJSON }, + activityPubOutbox, + staticFiles + } + } + + private async exportVideo (videoId: number) { + const [ video, captions, source ] = await Promise.all([ + VideoModel.loadFull(videoId), + VideoCaptionModel.listVideoCaptions(videoId), + VideoSourceModel.loadLatest(videoId) + ]) + + const passwords = video.privacy === VideoPrivacy.PASSWORD_PROTECTED + ? (await VideoPasswordModel.listPasswords({ videoId, start: 0, count: undefined, sort: 'createdAt' })).data + : [] + + const live = video.isLive + ? await VideoLiveModel.loadByVideoIdWithSettings(videoId) + : undefined; + + // We already have captions, so we can set it to the video object + (video as any).VideoCaptions = captions + // Then fetch more attributes for AP serialization + const videoAP = await video.lightAPToFullAP(undefined) + + const { relativePathsFromJSON, staticFiles } = this.exportVideoFiles({ video, captions }) + + return { + json: this.exportVideoJSON({ video, captions, live, passwords, source, archiveFiles: relativePathsFromJSON }), + staticFiles, + relativePathsFromJSON, + activityPubOutbox: await this.exportVideoAP(videoAP) + } + } + + // --------------------------------------------------------------------------- + + private exportVideoJSON (options: { + video: MVideoFullLight + captions: MVideoCaption[] + live: MVideoLiveWithSetting + passwords: MVideoPassword[] + source: MVideoSource + archiveFiles: VideoExportJSON['videos'][0]['archiveFiles'] + }): VideoExportJSON['videos'][0] { + const { video, captions, live, passwords, source, archiveFiles } = options + + return { + uuid: video.uuid, + + createdAt: video.createdAt.toISOString(), + updatedAt: video.updatedAt.toISOString(), + publishedAt: video.publishedAt.toISOString(), + originallyPublishedAt: video.originallyPublishedAt + ? video.originallyPublishedAt.toISOString() + : undefined, + + name: video.name, + category: video.category, + licence: video.licence, + language: video.language, + tags: video.Tags.map(t => t.name), + + privacy: video.privacy, + passwords: passwords.map(p => p.password), + + duration: video.duration, + + description: video.description, + support: video.support, + + isLive: video.isLive, + live: this.exportLiveJSON(video, live), + + url: video.url, + + thumbnailUrl: video.getMiniature()?.getOriginFileUrl(video) || null, + previewUrl: video.getPreview()?.getOriginFileUrl(video) || null, + + views: video.views, + + likes: video.likes, + dislikes: video.dislikes, + + nsfw: video.nsfw, + + commentsEnabled: video.commentsEnabled, + downloadEnabled: video.downloadEnabled, + + waitTranscoding: video.waitTranscoding, + state: video.state, + + channel: { + name: video.VideoChannel.Actor.preferredUsername + }, + + captions: this.exportCaptionsJSON(video, captions), + + files: this.exportFilesJSON(video, video.VideoFiles), + + streamingPlaylists: this.exportStreamingPlaylistsJSON(video, video.VideoStreamingPlaylists), + + source: source + ? { filename: source.filename } + : null, + + archiveFiles + } + } + + private exportLiveJSON (video: MVideo, live: MVideoLiveWithSetting) { + if (!video.isLive) return undefined + + return { + saveReplay: live.saveReplay, + permanentLive: live.permanentLive, + latencyMode: live.latencyMode, + streamKey: live.streamKey, + + replaySettings: live.ReplaySetting + ? { privacy: live.ReplaySetting.privacy } + : undefined + } + } + + private exportCaptionsJSON (video: MVideo, captions: MVideoCaption[]) { + return captions.map(c => ({ + createdAt: c.createdAt.toISOString(), + updatedAt: c.updatedAt.toISOString(), + language: c.language, + filename: c.filename, + fileUrl: c.getFileUrl(video) + })) + } + + private exportFilesJSON (video: MVideo, files: MVideoFile[]) { + return files.map(f => ({ + resolution: f.resolution, + size: f.size, + fps: f.fps, + + torrentUrl: f.getTorrentUrl(), + fileUrl: f.getFileUrl(video) + })) + } + + private exportStreamingPlaylistsJSON (video: MVideo, streamingPlaylists: MStreamingPlaylistFiles[]) { + return streamingPlaylists.map(p => ({ + type: p.type, + playlistUrl: p.getMasterPlaylistUrl(video), + segmentsSha256Url: p.getMasterPlaylistUrl(video), + files: this.exportFilesJSON(video, p.VideoFiles) + })) + } + + // --------------------------------------------------------------------------- + + private async exportVideoAP (video: MVideoAP): Promise> { + const videoFile = video.getMaxQualityFile() + const icon = video.getPreview() + + const videoFileAP = videoFile.toActivityPubObject(video) + + const audience = getAudience(video.VideoChannel.Account.Actor, video.privacy === VideoPrivacy.PUBLIC) + const videoObject = { + ...audiencify(await video.toActivityPubObject(), audience), + + icon: [ + { + ...icon.toActivityPubObject(video), + + url: join(this.options.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, icon)) + } + ], + + subtitleLanguage: video.VideoCaptions.map(c => ({ + ...c.toActivityPubObject(video), + + url: join(this.options.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, c)) + })), + + attachment: this.options.withVideoFiles + ? [ + { + type: 'Video' as 'Video', + url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile)), + + ...pick(videoFileAP, [ 'mediaType', 'height', 'size', 'fps' ]) + } + ] + : undefined + } + + return buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience) + } + + // --------------------------------------------------------------------------- + + private exportVideoFiles (options: { + video: MVideoFullLight + captions: MVideoCaption[] + }) { + const { video, captions } = options + + const staticFiles: ExportResult['staticFiles'] = [] + const relativePathsFromJSON = { + videoFile: null as string, + thumbnail: null as string, + captions: {} as { [ lang: string ]: string } + } + + const videoFile = video.getMaxQualityFile() + + if (this.options.withVideoFiles && videoFile) { + staticFiles.push({ + archivePath: this.getArchiveVideoFilePath(video, videoFile), + createrReadStream: () => this.generateVideoFileReadStream(video, videoFile) + }) + + relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile)) + } + + for (const caption of captions) { + staticFiles.push({ + archivePath: this.getArchiveCaptionFilePath(video, caption), + createrReadStream: () => Promise.resolve(createReadStream(caption.getFSPath())) + }) + + relativePathsFromJSON.captions[caption.language] = join(this.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, caption)) + } + + const thumbnail = video.getPreview() || video.getMiniature() + if (thumbnail) { + staticFiles.push({ + archivePath: this.getArchiveThumbnailFilePath(video, thumbnail), + createrReadStream: () => Promise.resolve(createReadStream(thumbnail.getPath())) + }) + + relativePathsFromJSON.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, thumbnail)) + } + + return { staticFiles, relativePathsFromJSON } + } + + private async generateVideoFileReadStream (video: MVideoFullLight, videoFile: MVideoFile): Promise { + if (videoFile.storage === FileStorage.FILE_SYSTEM) { + return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)) + } + + const { stream } = videoFile.isHLS() + ? await getHLSFileReadStream({ playlist: video.getHLSPlaylist(), filename: videoFile.filename, rangeHeader: undefined }) + : await getWebVideoFileReadStream({ filename: videoFile.filename, rangeHeader: undefined }) + + return stream + } + + private getArchiveVideoFilePath (video: MVideo, videoFile: MVideoFile) { + return join('video-files', video.uuid + extname(videoFile.filename)) + } + + private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) { + return join('captions', video.uuid + '-' + caption.language + extname(caption.filename)) + } + + private getArchiveThumbnailFilePath (video: MVideo, thumbnail: MThumbnail) { + return join('thumbnails', video.uuid + extname(thumbnail.filename)) + } +} diff --git a/server/core/lib/user-import-export/importers/abstract-rates-importer.ts b/server/core/lib/user-import-export/importers/abstract-rates-importer.ts new file mode 100644 index 000000000..a836570b6 --- /dev/null +++ b/server/core/lib/user-import-export/importers/abstract-rates-importer.ts @@ -0,0 +1,30 @@ +import { AbstractUserImporter } from './abstract-user-importer.js' +import { VideoRateType } from '@peertube/peertube-models' +import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js' +import { userRateVideo } from '@server/lib/rate.js' +import { VideoModel } from '@server/models/video/video.js' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' + +export abstract class AbstractRatesImporter extends AbstractUserImporter { + + protected sanitizeRate (data: O) { + if (!isUrlValid(data.videoUrl)) return undefined + + return data + } + + protected async importRate (data: { videoUrl: string }, rateType: VideoRateType) { + const videoUrl = data.videoUrl + const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl) + + if (!videoImmutable) { + throw new Error(`Cannot get or create video ${videoUrl} to import user ${rateType}`) + } + + const video = await VideoModel.loadFull(videoImmutable.id) + + await userRateVideo({ account: this.user.Account, rateType, video }) + + return { duplicate: false } + } +} diff --git a/server/core/lib/user-import-export/importers/abstract-user-importer.ts b/server/core/lib/user-import-export/importers/abstract-user-importer.ts new file mode 100644 index 000000000..ae301a689 --- /dev/null +++ b/server/core/lib/user-import-export/importers/abstract-user-importer.ts @@ -0,0 +1,119 @@ +import { getFileSize } from '@peertube/peertube-node-utils' +import { Awaitable } from '@peertube/peertube-typescript-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { MUserDefault } from '@server/types/models/user/user.js' +import { pathExists, readJSON, remove } from 'fs-extra/esm' +import { dirname, resolve } from 'path' + +const lTags = loggerTagsFactory('user-import') + +export abstract class AbstractUserImporter > }> { + protected user: MUserDefault + protected extractedDirectory: string + protected jsonFilePath: string + + constructor (options: { + user: MUserDefault + + extractedDirectory: string + + jsonFilePath: string + }) { + this.user = options.user + this.extractedDirectory = options.extractedDirectory + this.jsonFilePath = options.jsonFilePath + } + + getJSONFilePath () { + return this.jsonFilePath + } + + protected getSafeArchivePathOrThrow (path: string) { + if (!path) return undefined + + const resolved = resolve(dirname(this.jsonFilePath), path) + if (resolved.startsWith(this.extractedDirectory) !== true) { + throw new Error(`Static file path ${resolved} is outside the archive directory ${this.extractedDirectory}`) + } + + return resolved + } + + protected async cleanupImportedStaticFilePaths (archiveFiles: Record>) { + if (!archiveFiles || typeof archiveFiles !== 'object') return + + for (const file of Object.values(archiveFiles)) { + if (!file) continue + + try { + if (typeof file === 'string') { + await remove(this.getSafeArchivePathOrThrow(file)) + } else { // Avoid recursion to prevent security issue + for (const subFile of Object.values(file)) { + await remove(this.getSafeArchivePathOrThrow(subFile)) + } + } + } catch (err) { + logger.error(`Cannot remove file ${file} after successful import`, { err, ...lTags() }) + } + } + } + + protected async isFileValidOrLog (filePath: string, maxSize: number) { + if (!await pathExists(filePath)) { + logger.warn(`Do not import file ${filePath} that do not exist in zip`, lTags()) + return false + } + + const size = await getFileSize(filePath) + if (size > maxSize) { + logger.warn( + `Do not import too big file ${filePath} (${size} > ${maxSize})`, + lTags() + ) + return false + } + + return true + } + + async import () { + const importData: E = await readJSON(this.jsonFilePath) + const summary = { + duplicates: 0, + success: 0, + errors: 0 + } + + for (const importObject of this.getImportObjects(importData)) { + try { + const sanitized = this.sanitize(importObject) + + if (!sanitized) { + logger.warn('Do not import object after invalid sanitization', { importObject, ...lTags() }) + summary.errors++ + continue + } + + const result = await this.importObject(sanitized) + + await this.cleanupImportedStaticFilePaths(importObject.archiveFiles) + + if (result.duplicate === true) summary.duplicates++ + else summary.success++ + } catch (err) { + logger.error('Cannot import object from ' + this.jsonFilePath, { err, importObject, ...lTags() }) + + summary.errors++ + } + } + + return summary + } + + protected abstract getImportObjects (object: E): O[] + + protected abstract sanitize (object: O): O | undefined + + protected abstract importObject (object: O): Awaitable<{ duplicate: boolean }> +} diff --git a/server/core/lib/user-import-export/importers/account-blocklist-importer.ts b/server/core/lib/user-import-export/importers/account-blocklist-importer.ts new file mode 100644 index 000000000..844ef295a --- /dev/null +++ b/server/core/lib/user-import-export/importers/account-blocklist-importer.ts @@ -0,0 +1,66 @@ +import { BlocklistExportJSON } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { addAccountInBlocklist, addServerInBlocklist } from '@server/lib/blocklist.js' +import { ServerModel } from '@server/models/server/server.js' +import { AccountModel } from '@server/models/account/account.js' +import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js' +import { isHostValid } from '@server/helpers/custom-validators/servers.js' + +const lTags = loggerTagsFactory('user-import') + +type ImportObject = { handle: string | null, host: string | null, archiveFiles?: never } + +export class BlocklistImporter extends AbstractUserImporter { + + protected getImportObjects (json: BlocklistExportJSON) { + return [ + ...json.actors.map(o => ({ handle: o.handle, host: null })), + ...json.instances.map(o => ({ handle: null, host: o.host })) + ] + } + + protected sanitize (blocklistImportData: ImportObject) { + if (!isValidActorHandle(blocklistImportData.handle) && !isHostValid(blocklistImportData.host)) return undefined + + return blocklistImportData + } + + protected async importObject (blocklistImportData: ImportObject) { + if (blocklistImportData.handle) { + await this.importAccountBlock(blocklistImportData.handle) + } else { + await this.importServerBlock(blocklistImportData.host) + } + + return { duplicate: false } + } + + private async importAccountBlock (handle: string) { + const accountToBlock = await AccountModel.loadByNameWithHost(handle) + if (!accountToBlock) { + logger.info('Account %s was not blocked on user import because it cannot be found in the database.', handle, lTags()) + return + } + + await addAccountInBlocklist({ + byAccountId: this.user.Account.id, + targetAccountId: accountToBlock.id, + removeNotificationOfUserId: this.user.id + }) + + logger.info('Account %s blocked on user import.', handle, lTags()) + } + + private async importServerBlock (hostToBlock: string) { + const serverToBlock = await ServerModel.loadOrCreateByHost(hostToBlock) + + await addServerInBlocklist({ + byAccountId: this.user.Account.id, + targetServerId: serverToBlock.id, + removeNotificationOfUserId: this.user.id + }) + + logger.info('Server %s blocked on user import.', hostToBlock, lTags()) + } +} diff --git a/server/core/lib/user-import-export/importers/account-importer.ts b/server/core/lib/user-import-export/importers/account-importer.ts new file mode 100644 index 000000000..97632e001 --- /dev/null +++ b/server/core/lib/user-import-export/importers/account-importer.ts @@ -0,0 +1,54 @@ +import { AccountExportJSON, ActorImageType } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { updateLocalActorImageFiles } from '@server/lib/local-actor.js' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { MAccountDefault } from '@server/types/models/index.js' +import { isUserDescriptionValid, isUserDisplayNameValid } from '@server/helpers/custom-validators/users.js' + +const lTags = loggerTagsFactory('user-import') + +export class AccountImporter extends AbstractUserImporter { + + protected getImportObjects (json: AccountExportJSON) { + return [ json ] + } + + protected sanitize (blocklistImportData: AccountExportJSON) { + if (!isUserDisplayNameValid(blocklistImportData.name)) return undefined + + if (!isUserDescriptionValid(blocklistImportData.description)) blocklistImportData.description = null + + return blocklistImportData + } + + protected async importObject (accountImportData: AccountExportJSON) { + const account = this.user.Account + + account.name = accountImportData.displayName + account.description = accountImportData.description + + await saveInTransactionWithRetries(account) + + await this.importAvatar(account, accountImportData) + + logger.info('Account %s imported.', account.name, lTags()) + + return { duplicate: false } + } + + private async importAvatar (account: MAccountDefault, accountImportData: AccountExportJSON) { + const avatarPath = this.getSafeArchivePathOrThrow(accountImportData.archiveFiles.avatar) + if (!avatarPath) return undefined + + if (!await this.isFileValidOrLog(avatarPath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) return undefined + + await updateLocalActorImageFiles({ + accountOrChannel: account, + imagePhysicalFile: { path: avatarPath }, + type: ActorImageType.AVATAR, + sendActorUpdate: false + }) + } +} diff --git a/server/core/lib/user-import-export/importers/channels-importer.ts b/server/core/lib/user-import-export/importers/channels-importer.ts new file mode 100644 index 000000000..464c09ee2 --- /dev/null +++ b/server/core/lib/user-import-export/importers/channels-importer.ts @@ -0,0 +1,74 @@ +import { ActorImageType, ChannelExportJSON } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { pick } from '@peertube/peertube-core-utils' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { createLocalVideoChannel } from '@server/lib/video-channel.js' +import { JobQueue } from '@server/lib/job-queue/job-queue.js' +import { updateLocalActorImageFiles } from '@server/lib/local-actor.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { + isVideoChannelDescriptionValid, + isVideoChannelDisplayNameValid, + isVideoChannelSupportValid, + isVideoChannelUsernameValid +} from '@server/helpers/custom-validators/video-channels.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' + +const lTags = loggerTagsFactory('user-import') + +export class ChannelsImporter extends AbstractUserImporter { + + protected getImportObjects (json: ChannelExportJSON) { + return json.channels + } + + protected sanitize (blocklistImportData: ChannelExportJSON['channels'][0]) { + if (!isVideoChannelUsernameValid(blocklistImportData.name)) return undefined + if (!isVideoChannelDisplayNameValid(blocklistImportData.name)) return undefined + + if (!isVideoChannelDescriptionValid(blocklistImportData.description)) blocklistImportData.description = null + if (!isVideoChannelSupportValid(blocklistImportData.support)) blocklistImportData.description = null + + return blocklistImportData + } + + protected async importObject (channelImportData: ChannelExportJSON['channels'][0]) { + const account = this.user.Account + const existingChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(channelImportData.name) + + if (existingChannel) { + logger.info(`Do not import channel ${existingChannel.name} that already exists on this PeerTube instance`, lTags()) + } else { + const videoChannelCreated = await sequelizeTypescript.transaction(async t => { + return createLocalVideoChannel(pick(channelImportData, [ 'displayName', 'name', 'description', 'support' ]), account, t) + }) + + await JobQueue.Instance.createJob({ type: 'actor-keys', payload: { actorId: videoChannelCreated.actorId } }) + + for (const type of [ ActorImageType.AVATAR, ActorImageType.BANNER ]) { + const relativePath = type === ActorImageType.AVATAR + ? channelImportData.archiveFiles.avatar + : channelImportData.archiveFiles.banner + + if (!relativePath) continue + + const absolutePath = this.getSafeArchivePathOrThrow(relativePath) + if (!await this.isFileValidOrLog(absolutePath, CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max)) continue + + await updateLocalActorImageFiles({ + accountOrChannel: videoChannelCreated, + imagePhysicalFile: { path: absolutePath }, + type, + sendActorUpdate: false + }) + } + + logger.info('Video channel %s imported.', channelImportData.name, lTags()) + } + + return { + duplicate: !!existingChannel + } + } +} diff --git a/server/core/lib/user-import-export/importers/dislikes-importer.ts b/server/core/lib/user-import-export/importers/dislikes-importer.ts new file mode 100644 index 000000000..064d91e5a --- /dev/null +++ b/server/core/lib/user-import-export/importers/dislikes-importer.ts @@ -0,0 +1,17 @@ +import { DislikesExportJSON } from '@peertube/peertube-models' +import { AbstractRatesImporter } from './abstract-rates-importer.js' + +export class DislikesImporter extends AbstractRatesImporter { + + protected getImportObjects (json: DislikesExportJSON) { + return json.dislikes + } + + protected sanitize (o: DislikesExportJSON['dislikes'][0]) { + return this.sanitizeRate(o) + } + + protected async importObject (dislikesImportData: DislikesExportJSON['dislikes'][0]) { + return this.importRate(dislikesImportData, 'dislike') + } +} diff --git a/server/core/lib/user-import-export/importers/following-importer.ts b/server/core/lib/user-import-export/importers/following-importer.ts new file mode 100644 index 000000000..957a6c98b --- /dev/null +++ b/server/core/lib/user-import-export/importers/following-importer.ts @@ -0,0 +1,37 @@ +import { FollowingExportJSON } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { JobQueue } from '@server/lib/job-queue/job-queue.js' +import { isValidActorHandle } from '@server/helpers/custom-validators/activitypub/actor.js' + +const lTags = loggerTagsFactory('user-import') + +export class FollowingImporter extends AbstractUserImporter { + + protected getImportObjects (json: FollowingExportJSON) { + return json.following + } + + protected sanitize (followingImportData: FollowingExportJSON['following'][0]) { + if (!isValidActorHandle(followingImportData.targetHandle)) return undefined + + return followingImportData + } + + protected async importObject (followingImportData: FollowingExportJSON['following'][0]) { + const [ name, host ] = followingImportData.targetHandle.split('@') + + const payload = { + name, + host, + assertIsChannel: true, + followerActorId: this.user.Account.Actor.id + } + + await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload }) + + logger.info('Subscription job of %s created on user import.', followingImportData.targetHandle, lTags()) + + return { duplicate: false } + } +} diff --git a/server/core/lib/user-import-export/importers/index.ts b/server/core/lib/user-import-export/importers/index.ts new file mode 100644 index 000000000..fbfd723db --- /dev/null +++ b/server/core/lib/user-import-export/importers/index.ts @@ -0,0 +1,9 @@ +export * from './account-blocklist-importer.js' +export * from './account-importer.js' +export * from './channels-importer.js' +export * from './dislikes-importer.js' +export * from './following-importer.js' +export * from './likes-importer.js' +export * from './user-settings-importer.js' +export * from './video-playlists-importer.js' +export * from './videos-importer.js' diff --git a/server/core/lib/user-import-export/importers/likes-importer.ts b/server/core/lib/user-import-export/importers/likes-importer.ts new file mode 100644 index 000000000..70b56034d --- /dev/null +++ b/server/core/lib/user-import-export/importers/likes-importer.ts @@ -0,0 +1,17 @@ +import { LikesExportJSON } from '@peertube/peertube-models' +import { AbstractRatesImporter } from './abstract-rates-importer.js' + +export class LikesImporter extends AbstractRatesImporter { + + protected getImportObjects (json: LikesExportJSON) { + return json.likes + } + + protected sanitize (o: LikesExportJSON['likes'][0]) { + return this.sanitizeRate(o) + } + + protected async importObject (likesImportData: LikesExportJSON['likes'][0]) { + return this.importRate(likesImportData, 'like') + } +} diff --git a/server/core/lib/user-import-export/importers/user-settings-importer.ts b/server/core/lib/user-import-export/importers/user-settings-importer.ts new file mode 100644 index 000000000..5120993e1 --- /dev/null +++ b/server/core/lib/user-import-export/importers/user-settings-importer.ts @@ -0,0 +1,88 @@ +import { UserNotificationSetting, UserSettingsExportJSON } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { UserNotificationSettingModel } from '@server/models/user/user-notification-setting.js' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { + isUserAutoPlayNextVideoPlaylistValid, + isUserAutoPlayNextVideoValid, + isUserAutoPlayVideoValid, + isUserNSFWPolicyValid, + isUserP2PEnabledValid, + isUserVideoLanguages, + isUserVideosHistoryEnabledValid +} from '@server/helpers/custom-validators/users.js' +import { isThemeNameValid } from '@server/helpers/custom-validators/plugins.js' +import { isThemeRegistered } from '@server/lib/plugins/theme-utils.js' +import { isUserNotificationSettingValid } from '@server/helpers/custom-validators/user-notifications.js' + +const lTags = loggerTagsFactory('user-import') + +export class UserSettingsImporter extends AbstractUserImporter { + + protected getImportObjects (json: UserSettingsExportJSON) { + return [ json ] + } + + protected sanitize (o: UserSettingsExportJSON) { + if (!isUserNSFWPolicyValid(o.nsfwPolicy)) o.nsfwPolicy = undefined + + if (!isUserAutoPlayVideoValid(o.autoPlayVideo)) o.autoPlayVideo = undefined + if (!isUserAutoPlayNextVideoValid(o.autoPlayNextVideo)) o.autoPlayNextVideo = undefined + if (!isUserAutoPlayNextVideoPlaylistValid(o.autoPlayNextVideoPlaylist)) o.autoPlayNextVideoPlaylist = undefined + if (!isUserP2PEnabledValid(o.p2pEnabled)) o.p2pEnabled = undefined + if (!isUserVideosHistoryEnabledValid(o.videosHistoryEnabled)) o.videosHistoryEnabled = undefined + if (!isUserVideoLanguages(o.videoLanguages)) o.videoLanguages = undefined + if (!isThemeNameValid(o.theme) || !isThemeRegistered(o.theme)) o.theme = undefined + + for (const key of Object.keys(o.notificationSettings || {})) { + if (!isUserNotificationSettingValid(o.notificationSettings[key])) (o.notificationSettings[key] as any) = undefined + } + + return o + } + + protected async importObject (userImportData: UserSettingsExportJSON) { + if (exists(userImportData.nsfwPolicy)) this.user.nsfwPolicy = userImportData.nsfwPolicy + if (exists(userImportData.autoPlayVideo)) this.user.autoPlayVideo = userImportData.autoPlayVideo + if (exists(userImportData.autoPlayNextVideo)) this.user.autoPlayNextVideo = userImportData.autoPlayNextVideo + if (exists(userImportData.autoPlayNextVideoPlaylist)) this.user.autoPlayNextVideoPlaylist = userImportData.autoPlayNextVideoPlaylist + if (exists(userImportData.p2pEnabled)) this.user.p2pEnabled = userImportData.p2pEnabled + if (exists(userImportData.videosHistoryEnabled)) this.user.videosHistoryEnabled = userImportData.videosHistoryEnabled + if (exists(userImportData.videoLanguages)) this.user.videoLanguages = userImportData.videoLanguages + if (exists(userImportData.theme)) this.user.theme = userImportData.theme + + await saveInTransactionWithRetries(this.user) + + await this.updateSettings(userImportData.notificationSettings) + + logger.info('Settings of user %s imported.', this.user.username, lTags()) + + return { duplicate: false } + } + + private async updateSettings (settingsImportData: UserSettingsExportJSON['notificationSettings']) { + const values: UserNotificationSetting = { + newVideoFromSubscription: settingsImportData.newVideoFromSubscription, + newCommentOnMyVideo: settingsImportData.newCommentOnMyVideo, + myVideoImportFinished: settingsImportData.myVideoImportFinished, + myVideoPublished: settingsImportData.myVideoPublished, + abuseAsModerator: settingsImportData.abuseAsModerator, + videoAutoBlacklistAsModerator: settingsImportData.videoAutoBlacklistAsModerator, + blacklistOnMyVideo: settingsImportData.blacklistOnMyVideo, + newUserRegistration: settingsImportData.newUserRegistration, + commentMention: settingsImportData.commentMention, + newFollow: settingsImportData.newFollow, + newInstanceFollower: settingsImportData.newInstanceFollower, + abuseNewMessage: settingsImportData.abuseNewMessage, + abuseStateChange: settingsImportData.abuseStateChange, + autoInstanceFollowing: settingsImportData.autoInstanceFollowing, + newPeerTubeVersion: settingsImportData.newPeerTubeVersion, + newPluginVersion: settingsImportData.newPluginVersion, + myVideoStudioEditionFinished: settingsImportData.myVideoStudioEditionFinished + } + + await UserNotificationSettingModel.updateUserSettings(values, this.user.id) + } +} diff --git a/server/core/lib/user-import-export/importers/video-playlists-importer.ts b/server/core/lib/user-import-export/importers/video-playlists-importer.ts new file mode 100644 index 000000000..b89523318 --- /dev/null +++ b/server/core/lib/user-import-export/importers/video-playlists-importer.ts @@ -0,0 +1,169 @@ +import { VideoPlaylistPrivacy, VideoPlaylistType, VideoPlaylistsExportJSON } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { buildUUID } from '@peertube/peertube-node-utils' +import { + MChannelBannerAccountDefault, MVideoPlaylist, + MVideoPlaylistFull, + MVideoPlaylistThumbnail +} from '@server/types/models/index.js' +import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { updateLocalPlaylistMiniatureFromExisting } from '@server/lib/thumbnail.js' +import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js' +import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js' +import { + isVideoPlaylistDescriptionValid, + isVideoPlaylistNameValid, + isVideoPlaylistPrivacyValid, + isVideoPlaylistTimestampValid, + isVideoPlaylistTypeValid +} from '@server/helpers/custom-validators/video-playlists.js' +import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' + +const lTags = loggerTagsFactory('user-import') + +export class VideoPlaylistsImporter extends AbstractUserImporter { + + protected getImportObjects (json: VideoPlaylistsExportJSON) { + return json.videoPlaylists + } + + protected sanitize (o: VideoPlaylistsExportJSON['videoPlaylists'][0]) { + if (!isVideoPlaylistTypeValid(o.type)) return undefined + if (!isVideoPlaylistNameValid(o.displayName)) return undefined + if (!isVideoPlaylistPrivacyValid(o.privacy)) return undefined + if (!isArray(o.elements)) return undefined + + if (o.channel?.name && !isActorPreferredUsernameValid(o.channel.name)) o.channel = undefined + if (!isVideoPlaylistDescriptionValid(o.description)) o.description = undefined + + o.elements = o.elements.filter(e => { + if (!isUrlValid(e.videoUrl)) return false + if (e.startTimestamp && !isVideoPlaylistTimestampValid(e.startTimestamp)) return false + if (e.stopTimestamp && !isVideoPlaylistTimestampValid(e.stopTimestamp)) return false + + return true + }) + + return o + } + + protected async importObject (playlistImportData: VideoPlaylistsExportJSON['videoPlaylists'][0]) { + const existingPlaylist = await VideoPlaylistModel.loadRegularByAccountAndName(this.user.Account, playlistImportData.displayName) + + if (existingPlaylist) { + logger.info(`Do not import playlist ${playlistImportData.displayName} that already exists in the account`, lTags()) + return { duplicate: true } + } + + const videoPlaylist = playlistImportData.type === VideoPlaylistType.WATCH_LATER + ? await this.getWatchLaterPlaylist() + : await this.createPlaylist(playlistImportData) + + await this.createElements(videoPlaylist, playlistImportData) + + await sendCreateVideoPlaylist(videoPlaylist, undefined) + + logger.info('Video playlist %s imported.', videoPlaylist.name, lTags(videoPlaylist.uuid)) + + return { duplicate: false } + } + + private async createPlaylist (playlistImportData: VideoPlaylistsExportJSON['videoPlaylists'][0]) { + let videoChannel: MChannelBannerAccountDefault + + if (playlistImportData.channel.name) { + videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(playlistImportData.channel.name) + + if (!videoChannel) throw new Error(`Channel ${playlistImportData} not found`) + if (videoChannel.accountId !== this.user.Account.id) { + throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`) + } + } else if (playlistImportData.privacy !== VideoPlaylistPrivacy.PRIVATE) { + throw new Error('Cannot create a non private playlist without channel') + } + + const playlist: MVideoPlaylistFull = new VideoPlaylistModel({ + name: playlistImportData.displayName, + description: playlistImportData.description, + privacy: playlistImportData.privacy, + + uuid: buildUUID(), + videoChannelId: videoChannel?.id, + ownerAccountId: this.user.Account.id + }) + playlist.url = getLocalVideoPlaylistActivityPubUrl(playlist) + playlist.VideoChannel = videoChannel + playlist.OwnerAccount = this.user.Account + + await saveInTransactionWithRetries(playlist) + + await this.createThumbnail(playlist, playlistImportData) + + return playlist + } + + private async getWatchLaterPlaylist () { + return VideoPlaylistModel.loadWatchLaterOf(this.user.Account) + } + + private async createThumbnail (playlist: MVideoPlaylistThumbnail, playlistImportData: VideoPlaylistsExportJSON['videoPlaylists'][0]) { + const thumbnailPath = this.getSafeArchivePathOrThrow(playlistImportData.archiveFiles.thumbnail) + if (!thumbnailPath) return undefined + + if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.FILE_SIZE.max)) return undefined + + const thumbnail = await updateLocalPlaylistMiniatureFromExisting({ + inputPath: thumbnailPath, + playlist, + automaticallyGenerated: false + }) + + await playlist.setAndSaveThumbnail(thumbnail, undefined) + } + + private async createElements (playlist: MVideoPlaylist, playlistImportData: VideoPlaylistsExportJSON['videoPlaylists'][0]) { + const elementsToCreate: { videoId: number, startTimestamp: number, stopTimestamp: number }[] = [] + + for (const element of playlistImportData.elements.slice(0, USER_IMPORT.MAX_PLAYLIST_ELEMENTS)) { + const video = await loadOrCreateVideoIfAllowedForUser(element.videoUrl) + + if (!video) { + logger.debug(`Cannot get or create video ${element.videoUrl} to create playlist element in user import`, lTags()) + continue + } + + elementsToCreate.push({ + videoId: video.id, + startTimestamp: element.startTimestamp, + stopTimestamp: element.stopTimestamp + }) + } + + await sequelizeTypescript.transaction(async t => { + for (let position = 1; position <= elementsToCreate.length; position++) { + const elementToCreate = elementsToCreate[position - 1] + + const playlistElement = new VideoPlaylistElementModel({ + position, + startTimestamp: elementToCreate.startTimestamp, + stopTimestamp: elementToCreate.stopTimestamp, + videoPlaylistId: playlist.id, + videoId: elementToCreate.videoId + }) + await playlistElement.save({ transaction: t }) + + playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, playlistElement) + await playlistElement.save({ transaction: t }) + } + }) + } +} diff --git a/server/core/lib/user-import-export/importers/videos-importer.ts b/server/core/lib/user-import-export/importers/videos-importer.ts new file mode 100644 index 000000000..b02889794 --- /dev/null +++ b/server/core/lib/user-import-export/importers/videos-importer.ts @@ -0,0 +1,314 @@ +import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildNextVideoState } from '@server/lib/video-state.js' +import { VideoModel } from '@server/models/video/video.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID, getFileSize } from '@peertube/peertube-node-utils' +import { MChannelId, MThumbnail, MVideoCaption, MVideoFullLight } from '@server/types/models/index.js' +import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' +import { buildNewFile } from '@server/lib/video-file.js' +import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg' +import { updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { setVideoTags } from '@server/lib/video.js' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' +import { AbstractUserImporter } from './abstract-user-importer.js' +import { isUserQuotaValid } from '@server/lib/user.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { move } from 'fs-extra' +import { + isPasswordValid, + isVideoCategoryValid, + isVideoDescriptionValid, + isVideoDurationValid, + isVideoLanguageValid, + isVideoLicenceValid, + isVideoNameValid, + isVideoOriginallyPublishedAtValid, + isVideoPrivacyValid, + isVideoReplayPrivacyValid, + isVideoSupportValid, + isVideoTagValid +} from '@server/helpers/custom-validators/videos.js' +import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { CONFIG } from '@server/initializers/config.js' +import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js' +import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { parse } from 'path' +import { isLocalVideoFileAccepted } from '@server/lib/moderation.js' + +const lTags = loggerTagsFactory('user-import') + +export class VideosImporter extends AbstractUserImporter { + + protected getImportObjects (json: VideoExportJSON) { + return json.videos + } + + protected sanitize (o: VideoExportJSON['videos'][0]) { + if (!isVideoNameValid(o.name)) return undefined + if (!isVideoDurationValid(o.duration + '')) return undefined + if (!isVideoChannelUsernameValid(o.channel?.name)) return undefined + if (!isVideoPrivacyValid(o.privacy)) return undefined + if (!o.archiveFiles?.videoFile) return undefined + + if (!isVideoCategoryValid(o.category)) o.category = null + if (!isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE + if (!isVideoLanguageValid(o.language)) o.language = null + if (!isVideoDescriptionValid(o.description)) o.description = null + if (!isVideoSupportValid(o.support)) o.support = null + + if (!isBooleanValid(o.nsfw)) o.nsfw = false + if (!isBooleanValid(o.isLive)) o.isLive = false + if (!isBooleanValid(o.commentsEnabled)) o.commentsEnabled = CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED + if (!isBooleanValid(o.downloadEnabled)) o.downloadEnabled = CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED + if (!isBooleanValid(o.waitTranscoding)) o.waitTranscoding = true + + if (!isVideoOriginallyPublishedAtValid(o.originallyPublishedAt)) o.originallyPublishedAt = null + + if (!isArray(o.tags)) o.tags = [] + if (!isArray(o.captions)) o.captions = [] + + o.tags = o.tags.filter(t => isVideoTagValid(t)) + o.captions = o.captions.filter(c => isVideoCaptionLanguageValid(c.language)) + + if (o.isLive) { + if (!o.live) return undefined + if (!isBooleanValid(o.live.permanentLive)) return undefined + + if (!isBooleanValid(o.live.saveReplay)) o.live.saveReplay = false + if (o.live.saveReplay && !isVideoReplayPrivacyValid(o.live.replaySettings.privacy)) return undefined + + if (!isLiveLatencyModeValid(o.live.latencyMode)) o.live.latencyMode = LiveVideoLatencyMode.DEFAULT + + if (!o.live.streamKey) o.live.streamKey = buildUUID() + else if (!isUUIDValid(o.live.streamKey)) return undefined + } + + if (o.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + if (!isArray(o.passwords)) return undefined + // Refuse the import rather than handle only a portion of the passwords, which can be difficult for video owners to debug + if (o.passwords.some(p => !isPasswordValid(p))) return undefined + } + + return o + } + + protected async importObject (videoImportData: VideoExportJSON['videos'][0]) { + const videoFilePath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.videoFile) + const videoSize = await getFileSize(videoFilePath) + + if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) { + throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`) + } + + const videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(videoImportData.channel.name) + if (!videoChannel) throw new Error(`Channel ${videoImportData} not found`) + if (videoChannel.accountId !== this.user.Account.id) { + throw new Error(`Channel ${videoChannel.name} is not owned by user ${this.user.username}`) + } + + const existingVideo = await VideoModel.loadByNameAndChannel(videoChannel, videoImportData.name) + if (existingVideo && Math.abs(existingVideo.duration - videoImportData.duration) <= 1) { + logger.info(`Do not import video ${videoImportData.name} that already exists in the account`, lTags()) + return { duplicate: true } + } + + const ffprobe = await ffprobePromise(videoFilePath) + const duration = await getVideoStreamDuration(videoFilePath, ffprobe) + const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video', ffprobe }) + + await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoFile.size, channel: videoChannel, videoImportData }) + + let videoData = { + ...pick(videoImportData, [ + 'name', + 'category', + 'licence', + 'language', + 'privacy', + 'description', + 'support', + 'isLive', + 'nsfw', + 'commentsEnabled', + 'downloadEnabled', + 'waitTranscoding' + ]), + + uuid: buildUUID(), + duration, + remote: false, + state: buildNextVideoState(), + channelId: videoChannel.id, + originallyPublishedAt: videoImportData.originallyPublishedAt + ? new Date(videoImportData.originallyPublishedAt) + : undefined + } + + videoData = await Hooks.wrapObject(videoData, 'filter:api.video.user-import.video-attribute.result') + + const video = new VideoModel(videoData) as MVideoFullLight + video.VideoChannel = videoChannel + video.url = getLocalVideoActivityPubUrl(video) + + const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) + await move(videoFilePath, destination) + + const thumbnailPath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.thumbnail) + + const thumbnails: MThumbnail[] = [] + for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { + if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max)) continue + + thumbnails.push( + await updateLocalVideoMiniatureFromExisting({ + inputPath: thumbnailPath, + video, + type, + automaticallyGenerated: false, + keepOriginal: true + }) + ) + } + + const { videoCreated } = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight + + for (const thumbnail of thumbnails) { + await videoCreated.addAndSaveThumbnail(thumbnail, t) + } + + videoFile.videoId = video.id + await videoFile.save(sequelizeOptions) + + video.VideoFiles = [ videoFile ] + + await setVideoTags({ video, tags: videoImportData.tags, transaction: t }) + + await autoBlacklistVideoIfNeeded({ + video, + user: this.user, + isRemote: false, + isNew: true, + isNewFile: true, + transaction: t + }) + + if (videoImportData.source?.filename) { + await VideoSourceModel.create({ + filename: videoImportData.source.filename, + videoId: video.id + }, { transaction: t }) + } + + if (videoImportData.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + await VideoPasswordModel.addPasswords(videoImportData.passwords, video.id, t) + } + + if (videoImportData.isLive) { + const videoLive = new VideoLiveModel(pick(videoImportData.live, [ 'saveReplay', 'permanentLive', 'latencyMode', 'streamKey' ])) + + if (videoLive.saveReplay) { + const replaySettings = new VideoLiveReplaySettingModel({ + privacy: videoImportData.live.replaySettings.privacy + }) + await replaySettings.save(sequelizeOptions) + + videoLive.replaySettingId = replaySettings.id + } + + videoLive.videoId = videoCreated.id + videoCreated.VideoLive = await videoLive.save(sequelizeOptions) + } + + return { videoCreated } + }) + + await this.importCaptions(videoCreated, videoImportData) + + await addVideoJobsAfterCreation({ video: videoCreated, videoFile }) + + logger.info('Video %s imported.', video.name, lTags(videoCreated.uuid)) + + return { duplicate: false } + } + + private async importCaptions (video: MVideoFullLight, videoImportData: VideoExportJSON['videos'][0]) { + const captionPaths: string[] = [] + + for (const captionImport of videoImportData.captions) { + const relativeFilePath = videoImportData.archiveFiles?.captions?.[captionImport.language] + + if (!relativeFilePath) { + logger.warn('Cannot import caption ' + captionImport.language + ': file does not exist in the archive', lTags(video.uuid)) + continue + } + + const absoluteFilePath = this.getSafeArchivePathOrThrow(relativeFilePath) + + if (!await this.isFileValidOrLog(absoluteFilePath, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)) continue + + const videoCaption = new VideoCaptionModel({ + videoId: video.id, + filename: VideoCaptionModel.generateCaptionName(captionImport.language), + language: captionImport.language + }) as MVideoCaption + + await moveAndProcessCaptionFile({ path: absoluteFilePath }, videoCaption) + + await sequelizeTypescript.transaction(async (t) => { + await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) + }) + + captionPaths.push(absoluteFilePath) + } + + return captionPaths + } + + private async checkVideoFileIsAcceptedOrThrow (options: { + videoFilePath: string + size: number + channel: MChannelId + videoImportData: VideoExportJSON['videos'][0] + }) { + const { videoFilePath, size, videoImportData, channel } = options + + // Check we accept this video + const acceptParameters = { + videoBody: { + ...videoImportData, + + channelId: channel.id + }, + videoFile: { + path: videoFilePath, + filename: parse(videoFilePath).name, + size, + originalname: null + }, + user: this.user + } + const acceptedResult = await Hooks.wrapFun(isLocalVideoFileAccepted, acceptParameters, 'filter:api.video.user-import.accept.result') + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused local video file to import.', { acceptedResult, acceptParameters, ...lTags() }) + + throw new Error('Video file is not accepted: ' + acceptedResult.errorMessage || 'unknown reason') + } + } +} diff --git a/server/core/lib/user-import-export/user-exporter.ts b/server/core/lib/user-import-export/user-exporter.ts new file mode 100644 index 000000000..30c6fb38b --- /dev/null +++ b/server/core/lib/user-import-export/user-exporter.ts @@ -0,0 +1,282 @@ +import { join, parse } from 'path' +import { + AccountExporter, + BlocklistExporter, + ChannelsExporter, + CommentsExporter, + DislikesExporter, + ExportResult, + FollowersExporter, + FollowingExporter, + LikesExporter, AbstractUserExporter, + UserSettingsExporter, + VideoPlaylistsExporter, + VideosExporter +} from './exporters/index.js' +import { MUserDefault, MUserExport } from '@server/types/models/index.js' +import archiver, { Archiver } from 'archiver' +import { createWriteStream } from 'fs' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { PassThrough, Readable, Writable } from 'stream' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { getContextFilter } from '../activitypub/context.js' +import { activityPubCollection } from '../activitypub/collection.js' +import { FileStorage, UserExportState } from '@peertube/peertube-models' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { UserModel } from '@server/models/user/user.js' +import { getFSUserExportFilePath } from '../paths.js' +import { getUserExportFileObjectStorageSize, removeUserExportObjectStorage, storeUserExportFile } from '../object-storage/user-export.js' +import { getFileSize } from '@peertube/peertube-node-utils' +import { remove } from 'fs-extra/esm' + +const lTags = loggerTagsFactory('user-export') + +export class UserExporter { + + private archive: Archiver + + async export (exportModel: MUserExport) { + try { + exportModel.state = UserExportState.PROCESSING + await saveInTransactionWithRetries(exportModel) + + const user = await UserModel.loadByIdFull(exportModel.userId) + + let endPromise: Promise + let output: Writable + + if (exportModel.storage === FileStorage.FILE_SYSTEM) { + output = createWriteStream(getFSUserExportFilePath(exportModel)) + endPromise = new Promise(res => output.on('close', () => res())) + } else { + output = new PassThrough() + endPromise = storeUserExportFile(output as PassThrough, exportModel) + } + + await this.createZip({ exportModel, user, output }) + + await endPromise + + exportModel.state = UserExportState.COMPLETED + exportModel.size = exportModel.storage === FileStorage.FILE_SYSTEM + ? await getFileSize(getFSUserExportFilePath(exportModel)) + : await getUserExportFileObjectStorageSize(exportModel) + + await saveInTransactionWithRetries(exportModel) + } catch (err) { + logger.error('Cannot generate an export', { err, ...lTags() }) + + try { + exportModel.state = UserExportState.ERRORED + exportModel.error = err.message + + await saveInTransactionWithRetries(exportModel) + } catch (innerErr) { + logger.error('Cannot set export error state', { err: innerErr, ...lTags() }) + } + + try { + if (exportModel.storage === FileStorage.FILE_SYSTEM) { + await remove(getFSUserExportFilePath(exportModel)) + } else { + await removeUserExportObjectStorage(exportModel) + } + } catch (innerErr) { + logger.error('Cannot remove archive path after failure', { err: innerErr, ...lTags() }) + } + + throw err + } + } + + private createZip (options: { + exportModel: MUserExport + user: MUserDefault + output: Writable + }) { + const { output, exportModel, user } = options + + let activityPubOutboxStore: ExportResult['activityPubOutbox'] = [] + + this.archive = archiver('zip', { + zlib: { + level: 9 + } + }) + + return new Promise(async (res, rej) => { + this.archive.on('warning', err => { + logger.warn('Warning to archive a file in ' + exportModel.filename, { err }) + }) + + this.archive.on('error', err => { + rej(err) + }) + + this.archive.pipe(output) + + try { + for (const { exporter, jsonFilename } of this.buildExporters(exportModel, user)) { + const { json, staticFiles, activityPub, activityPubOutbox } = await exporter.export() + + logger.debug('Adding JSON file ' + jsonFilename + ' in archive ' + exportModel.filename) + this.appendJSON(json, join('peertube', jsonFilename)) + + if (activityPub) { + const activityPubFilename = exporter.getActivityPubFilename() + if (!activityPubFilename) throw new Error('ActivityPub filename is required for exporter that export activity pub data') + + this.appendJSON(activityPub, join('activity-pub', activityPubFilename)) + } + + if (activityPubOutbox) { + activityPubOutboxStore = activityPubOutboxStore.concat(activityPubOutbox) + } + + for (const file of staticFiles) { + const archivePath = join('files', parse(jsonFilename).name, file.archivePath) + + logger.debug(`Adding static file ${archivePath} in archive`) + + try { + await this.addToArchiveAndWait(await file.createrReadStream(), archivePath) + } catch (err) { + logger.error(`Cannot add ${archivePath} in archive`, { err }) + } + } + } + + this.appendJSON( + await activityPubContextify(activityPubCollection('outbox.json', activityPubOutboxStore), 'Video', getContextFilter()), + join('activity-pub', 'outbox.json') + ) + + await this.archive.finalize() + + res() + } catch (err) { + this.archive.abort() + + rej(err) + } + }) + } + + private buildExporters (exportModel: MUserExport, user: MUserDefault) { + const options = { + user, + activityPubFilenames: { + dislikes: 'dislikes.json', + likes: 'likes.json', + outbox: 'outbox.json', + following: 'following.json', + account: 'actor.json' + } + } + + return [ + { + jsonFilename: 'videos.json', + + exporter: new VideosExporter({ + ...options, + + relativeStaticDirPath: '../files/videos', + withVideoFiles: exportModel.withVideoFiles + }) + }, + { + jsonFilename: 'channels.json', + exporter: new ChannelsExporter({ + ...options, + + relativeStaticDirPath: '../files/channels' + }) + }, + { + jsonFilename: 'account.json', + exporter: new AccountExporter({ + ...options, + + relativeStaticDirPath: '../files/account' + }) + }, + { + jsonFilename: 'blocklist.json', + exporter: new BlocklistExporter(options) + }, + { + jsonFilename: 'likes.json', + exporter: new LikesExporter(options) + }, + { + jsonFilename: 'dislikes.json', + exporter: new DislikesExporter(options) + }, + { + jsonFilename: 'follower.json', + exporter: new FollowersExporter(options) + }, + { + jsonFilename: 'following.json', + exporter: new FollowingExporter(options) + }, + { + jsonFilename: 'user-settings.json', + exporter: new UserSettingsExporter(options) + }, + { + jsonFilename: 'comments.json', + exporter: new CommentsExporter(options) + }, + { + jsonFilename: 'video-playlists.json', + exporter: new VideoPlaylistsExporter({ + ...options, + + relativeStaticDirPath: '../files/video-playlists' + }) + } + ] as { jsonFilename: string, exporter: AbstractUserExporter }[] + } + + private addToArchiveAndWait (stream: Readable, archivePath: string) { + let errored = false + + return new Promise((res, rej) => { + const self = this + + function cleanup () { + self.archive.off('entry', entryListener) + } + + function entryListener ({ name }) { + if (name !== archivePath) return + + cleanup() + + return res() + } + + stream.once('error', err => { + cleanup() + + errored = true + return rej(err) + }) + + this.archive.on('entry', entryListener) + + // Prevent sending a stream that has an error on open resulting in a stucked archiving process + stream.once('readable', () => { + if (errored) return + + this.archive.append(stream, { name: archivePath }) + }) + }) + } + + private appendJSON (json: any, name: string) { + this.archive.append(JSON.stringify(json, undefined, 2), { name }) + } +} diff --git a/server/core/lib/user-import-export/user-importer.ts b/server/core/lib/user-import-export/user-importer.ts new file mode 100644 index 000000000..731bff5a4 --- /dev/null +++ b/server/core/lib/user-import-export/user-importer.ts @@ -0,0 +1,145 @@ +import { MUserDefault, MUserImport } from '@server/types/models/index.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { UserImportResultSummary, UserImportState } from '@peertube/peertube-models' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { getFSUserImportFilePath } from '../paths.js' +import { remove } from 'fs-extra/esm' +import { unzip } from '@server/helpers/unzip.js' +import { getFilenameWithoutExt } from '@peertube/peertube-node-utils' +import { VideosImporter } from './importers/videos-importer.js' +import { UserModel } from '@server/models/user/user.js' +import { dirname, join } from 'path' +import { AccountImporter } from './importers/account-importer.js' +import { UserSettingsImporter } from './importers/user-settings-importer.js' +import { ChannelsImporter } from './importers/channels-importer.js' +import { BlocklistImporter } from './importers/account-blocklist-importer.js' +import { FollowingImporter } from './importers/following-importer.js' +import { LikesImporter } from './importers/likes-importer.js' +import { DislikesImporter } from './importers/dislikes-importer.js' +import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js' + +const lTags = loggerTagsFactory('user-import') + +export class UserImporter { + private extractedDirectory: string + + async import (importModel: MUserImport) { + const resultSummary: UserImportResultSummary = { + stats: { + blocklist: this.buildSummary(), + channels: this.buildSummary(), + likes: this.buildSummary(), + dislikes: this.buildSummary(), + following: this.buildSummary(), + videoPlaylists: this.buildSummary(), + videos: this.buildSummary(), + account: this.buildSummary(), + userSettings: this.buildSummary() + } + } + + try { + importModel.state = UserImportState.PROCESSING + await saveInTransactionWithRetries(importModel) + + const inputZip = getFSUserImportFilePath(importModel) + this.extractedDirectory = join(dirname(inputZip), getFilenameWithoutExt(inputZip)) + + await unzip(inputZip, this.extractedDirectory) + + const user = await UserModel.loadByIdFull(importModel.userId) + + for (const { name, importer } of this.buildImporters(user)) { + try { + const { duplicates, errors, success } = await importer.import() + + resultSummary.stats[name].duplicates += duplicates + resultSummary.stats[name].errors += errors + resultSummary.stats[name].success += success + } catch (err) { + logger.error(`Cannot import ${importer.getJSONFilePath()} from ${inputZip}`, { err, ...lTags() }) + + resultSummary.stats[name].errors++ + } + } + + importModel.state = UserImportState.COMPLETED + importModel.resultSummary = resultSummary + await saveInTransactionWithRetries(importModel) + } catch (err) { + logger.error('Cannot import user archive', { toto: 'coucou', err, ...lTags() }) + + try { + importModel.state = UserImportState.ERRORED + importModel.error = err.message + + await saveInTransactionWithRetries(importModel) + } catch (innerErr) { + logger.error('Cannot set import error state', { err: innerErr, ...lTags() }) + } + + throw err + } finally { + try { + await remove(getFSUserImportFilePath(importModel)) + await remove(this.extractedDirectory) + } catch (innerErr) { + logger.error('Cannot remove import archive and directory after failure', { err: innerErr, ...lTags() }) + } + } + } + + private buildImporters (user: MUserDefault) { + // Keep consistency in import order (don't import videos before channels for example) + return [ + { + name: 'account' as 'account', + importer: new AccountImporter(this.buildImporterOptions(user, 'account.json')) + }, + { + name: 'userSettings' as 'userSettings', + importer: new UserSettingsImporter(this.buildImporterOptions(user, 'user-settings.json')) + }, + { + name: 'channels' as 'channels', + importer: new ChannelsImporter(this.buildImporterOptions(user, 'channels.json')) + }, + { + name: 'blocklist' as 'blocklist', + importer: new BlocklistImporter(this.buildImporterOptions(user, 'blocklist.json')) + }, + { + name: 'following' as 'following', + importer: new FollowingImporter(this.buildImporterOptions(user, 'following.json')) + }, + { + name: 'videos' as 'videos', + importer: new VideosImporter(this.buildImporterOptions(user, 'videos.json')) + }, + { + name: 'likes' as 'likes', + importer: new LikesImporter(this.buildImporterOptions(user, 'likes.json')) + }, + { + name: 'dislikes' as 'dislikes', + importer: new DislikesImporter(this.buildImporterOptions(user, 'dislikes.json')) + }, + { + name: 'videoPlaylists' as 'videoPlaylists', + importer: new VideoPlaylistsImporter(this.buildImporterOptions(user, 'video-playlists.json')) + } + ] + } + + private buildImporterOptions (user: MUserDefault, jsonFilename: string) { + return { + extractedDirectory: this.extractedDirectory, + user, + jsonFilePath: join(this.extractedDirectory, 'peertube', jsonFilename) + } + } + + private buildSummary () { + return { success: 0, duplicates: 0, errors: 0 } + } +} diff --git a/server/core/lib/user.ts b/server/core/lib/user.ts index 5e5970e79..ba78aba59 100644 --- a/server/core/lib/user.ts +++ b/server/core/lib/user.ts @@ -196,33 +196,24 @@ async function sendVerifyRegistrationEmail (registration: MRegistration) { // --------------------------------------------------------------------------- async function getOriginalVideoFileTotalFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: false - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) + const base = await UserModel.getUserQuota({ userId: user.id, daily: false }) return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) } // Returns cumulative size of all video files uploaded in the last 24 hours. async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: true - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) + const base = await UserModel.getUserQuota({ userId: user.id, daily: true }) return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) } -async function isAbleToUploadVideo (userId: number, newVideoSize: number) { +async function isUserQuotaValid (options: { + userId: number + uploadSize: number + checkDaily?: boolean // default true +}) { + const { userId, uploadSize, checkDaily = true } = options const user = await UserModel.loadById(userId) if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) @@ -232,18 +223,18 @@ async function isAbleToUploadVideo (userId: number, newVideoSize: number) { getOriginalVideoFileTotalDailyFromUser(user) ]) - const uploadedTotal = newVideoSize + totalBytes - const uploadedDaily = newVideoSize + totalBytesDaily + const uploadedTotal = uploadSize + totalBytes + const uploadedDaily = uploadSize + totalBytesDaily logger.debug( - 'Check user %d quota to upload another video.', userId, - { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, newVideoSize } + 'Check user %d quota to upload content.', userId, + { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, uploadSize } ) - if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota - if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily + if (checkDaily && user.videoQuotaDaily !== -1 && uploadedDaily >= user.videoQuotaDaily) return false + if (user.videoQuota !== -1 && uploadedTotal >= user.videoQuota) return false - return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily + return true } // --------------------------------------------------------------------------- @@ -258,7 +249,7 @@ export { sendVerifyUserEmail, sendVerifyRegistrationEmail, - isAbleToUploadVideo, + isUserQuotaValid, buildUser } diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts new file mode 100644 index 000000000..d69cda8a3 --- /dev/null +++ b/server/core/lib/video-jobs.ts @@ -0,0 +1,164 @@ +import { ManageVideoTorrentPayload, VideoPrivacy, VideoPrivacyType, VideoState, VideoStateType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MVideo, MVideoFile, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js' +import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue.js' +import { moveFilesIfPrivacyChanged } from './video-privacy.js' + +export async function buildMoveJob (options: { + video: MVideoUUID + previousVideoState: VideoStateType + type: 'move-to-object-storage' | 'move-to-file-system' + isNewVideo?: boolean // Default true +}) { + const { video, previousVideoState, isNewVideo = true, type } = options + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') + + return { + type, + payload: { + videoUUID: video.uuid, + isNewVideo, + previousVideoState + } + } +} + +export function buildStoryboardJobIfNeeded (options: { + video: MVideo + federate: boolean +}) { + const { video, federate } = options + + if (CONFIG.STORYBOARDS.ENABLED) { + return { + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate + } + } + } + + if (federate === true) { + return { + type: 'federate-video' as 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideoForFederation: false + } + } + } + + return undefined +} + +export async function addVideoJobsAfterCreation (options: { + video: MVideo + videoFile: MVideoFile +}) { + const { video, videoFile } = options + + const jobs: (CreateJobArgument & CreateJobOptions)[] = [ + { + type: 'manage-video-torrent' as 'manage-video-torrent', + payload: { + videoId: video.id, + videoFileId: videoFile.id, + action: 'create' + } + }, + + buildStoryboardJobIfNeeded({ video, federate: false }), + + { + type: 'notify', + payload: { + action: 'new-video', + videoUUID: video.uuid + } + }, + + { + type: 'federate-video' as 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideoForFederation: true + } + } + ] + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' })) + } + + if (video.state === VideoState.TO_TRANSCODE) { + jobs.push({ + type: 'transcoding-job-builder' as 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + optimizeJob: { + isNewVideo: true + } + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} + +export async function addVideoJobsAfterUpdate (options: { + video: MVideoFullLight + isNewVideoForFederation: boolean + + nameChanged: boolean + oldPrivacy: VideoPrivacyType +}) { + const { video, nameChanged, oldPrivacy, isNewVideoForFederation } = options + const jobs: CreateJobArgument[] = [] + + const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) + + if (!video.isLive && (nameChanged || filePathChanged)) { + for (const file of (video.VideoFiles || [])) { + const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } + + jobs.push({ type: 'manage-video-torrent', payload }) + } + + const hls = video.getHLSPlaylist() + + for (const file of (hls?.VideoFiles || [])) { + const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } + + jobs.push({ type: 'manage-video-torrent', payload }) + } + } + + jobs.push({ + type: 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideoForFederation + } + }) + + const wasConfidentialVideo = new Set([ + VideoPrivacy.PRIVATE, + VideoPrivacy.UNLISTED, + VideoPrivacy.INTERNAL + ]).has(oldPrivacy) + + if (wasConfidentialVideo) { + jobs.push({ + type: 'notify', + payload: { + action: 'new-video', + videoUUID: video.uuid + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} diff --git a/server/core/lib/video-path-manager.ts b/server/core/lib/video-path-manager.ts index 0a8604a58..ecf8ccbc6 100644 --- a/server/core/lib/video-path-manager.ts +++ b/server/core/lib/video-path-manager.ts @@ -1,7 +1,7 @@ import { Mutex } from 'async-mutex' import { remove } from 'fs-extra/esm' import { extname, join } from 'path' -import { VideoStorage } from '@peertube/peertube-models' +import { FileStorage } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { extractVideo } from '@server/helpers/video.js' import { CONFIG } from '@server/initializers/config.js' @@ -63,7 +63,7 @@ class VideoPathManager { } async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { + if (videoFile.storage === FileStorage.FILE_SYSTEM) { return this.makeAvailableFactory( () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), false, @@ -93,7 +93,7 @@ class VideoPathManager { async makeAvailableResolutionPlaylistFile (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { const filename = getHlsResolutionPlaylistFilename(videoFile.filename) - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { + if (videoFile.storage === FileStorage.FILE_SYSTEM) { return this.makeAvailableFactory( () => join(getHLSDirectory(videoFile.getVideo()), filename), false, @@ -110,7 +110,7 @@ class VideoPathManager { } async makeAvailablePlaylistFile (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB) { - if (playlist.storage === VideoStorage.FILE_SYSTEM) { + if (playlist.storage === FileStorage.FILE_SYSTEM) { return this.makeAvailableFactory( () => join(getHLSDirectory(playlist.Video), filename), false, diff --git a/server/core/lib/video-privacy.ts b/server/core/lib/video-privacy.ts index e7594c173..d32643da3 100644 --- a/server/core/lib/video-privacy.ts +++ b/server/core/lib/video-privacy.ts @@ -1,6 +1,6 @@ import { move } from 'fs-extra/esm' import { join } from 'path' -import { VideoPrivacy, VideoPrivacyType, VideoStorage } from '@peertube/peertube-models' +import { VideoPrivacy, VideoPrivacyType, FileStorage } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' import { DIRECTORIES } from '@server/initializers/constants.js' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' @@ -66,7 +66,7 @@ async function moveFiles (options: { const { type, video } = options for (const file of video.VideoFiles) { - if (file.storage === VideoStorage.FILE_SYSTEM) { + if (file.storage === FileStorage.FILE_SYSTEM) { await moveWebVideoFileOnFS(type, video, file) } else { await updateWebVideoFileACL(video, file) @@ -76,7 +76,7 @@ async function moveFiles (options: { const hls = video.getHLSPlaylist() if (hls) { - if (hls.storage === VideoStorage.FILE_SYSTEM) { + if (hls.storage === FileStorage.FILE_SYSTEM) { await moveHLSFilesOnFS(type, video) } else { await updateHLSFilesACL(hls) diff --git a/server/core/lib/video-state.ts b/server/core/lib/video-state.ts index 069140ef6..d360a49ca 100644 --- a/server/core/lib/video-state.ts +++ b/server/core/lib/video-state.ts @@ -10,7 +10,7 @@ import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index. import { federateVideoIfNeeded } from './activitypub/videos/index.js' import { JobQueue } from './job-queue/index.js' import { Notifier } from './notifier/index.js' -import { buildMoveJob } from './video.js' +import { buildMoveJob } from './video-jobs.js' function buildNextVideoState (currentState?: VideoStateType) { if (currentState === VideoState.PUBLISHED) { diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.ts index fbf74103d..6e118bd00 100644 --- a/server/core/lib/video-studio.ts +++ b/server/core/lib/video-studio.ts @@ -11,7 +11,7 @@ import { VideoStudioTranscodingJobHandler } from './runners/index.js' import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js' import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js' import { VideoPathManager } from './video-path-manager.js' -import { buildStoryboardJobIfNeeded } from './video.js' +import { buildStoryboardJobIfNeeded } from './video-jobs.js' const lTags = loggerTagsFactory('video-studio') diff --git a/server/core/lib/video.ts b/server/core/lib/video.ts index d9e55d712..0df9f58d6 100644 --- a/server/core/lib/video.ts +++ b/server/core/lib/video.ts @@ -2,24 +2,18 @@ import { UploadFiles } from 'express' import memoizee from 'memoizee' import { Transaction } from 'sequelize' import { - ManageVideoTorrentPayload, ThumbnailType, ThumbnailType_Type, VideoCreate, - VideoPrivacy, - VideoPrivacyType, - VideoStateType + VideoPrivacy } from '@peertube/peertube-models' import { CONFIG } from '@server/initializers/config.js' import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants.js' import { TagModel } from '@server/models/video/tag.js' -import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' import { VideoModel } from '@server/models/video/video.js' import { FilteredModelAttributes } from '@server/types/index.js' -import { MThumbnail, MVideo, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models/index.js' -import { CreateJobArgument, JobQueue } from './job-queue/job-queue.js' +import { MThumbnail, MVideoTag, MVideoThumbnail } from '@server/types/models/index.js' import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js' -import { moveFilesIfPrivacyChanged } from './video-privacy.js' export function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes { return { @@ -95,59 +89,6 @@ export async function setVideoTags (options: { // --------------------------------------------------------------------------- -export async function buildMoveJob (options: { - video: MVideoUUID - previousVideoState: VideoStateType - type: 'move-to-object-storage' | 'move-to-file-system' - isNewVideo?: boolean // Default true -}) { - const { video, previousVideoState, isNewVideo = true, type } = options - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') - - return { - type, - payload: { - videoUUID: video.uuid, - isNewVideo, - previousVideoState - } - } -} - -// --------------------------------------------------------------------------- - -export function buildStoryboardJobIfNeeded (options: { - video: MVideo - federate: boolean -}) { - const { video, federate } = options - - if (CONFIG.STORYBOARDS.ENABLED) { - return { - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - federate - } - } - } - - if (federate === true) { - return { - type: 'federate-video' as 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideoForFederation: false - } - } - } - - return undefined -} - -// --------------------------------------------------------------------------- - export async function getVideoDuration (videoId: number | string) { const video = await VideoModel.load(videoId) @@ -163,60 +104,3 @@ export const getCachedVideoDuration = memoizee(getVideoDuration, { max: MEMOIZE_LENGTH.VIDEO_DURATION, maxAge: MEMOIZE_TTL.VIDEO_DURATION }) - -// --------------------------------------------------------------------------- - -export async function addVideoJobsAfterUpdate (options: { - video: MVideoFullLight - isNewVideoForFederation: boolean - - nameChanged: boolean - oldPrivacy: VideoPrivacyType -}) { - const { video, nameChanged, oldPrivacy, isNewVideoForFederation } = options - const jobs: CreateJobArgument[] = [] - - const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) - - if (!video.isLive && (nameChanged || filePathChanged)) { - for (const file of (video.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - - const hls = video.getHLSPlaylist() - - for (const file of (hls?.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - } - - jobs.push({ - type: 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideoForFederation - } - }) - - const wasConfidentialVideo = new Set([ - VideoPrivacy.PRIVATE, - VideoPrivacy.UNLISTED, - VideoPrivacy.INTERNAL - ]).has(oldPrivacy) - - if (wasConfidentialVideo) { - jobs.push({ - type: 'notify', - payload: { - action: 'new-video', - videoUUID: video.uuid - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} diff --git a/server/core/middlewares/validators/config.ts b/server/core/middlewares/validators/config.ts index e495bb959..b9477b75e 100644 --- a/server/core/middlewares/validators/config.ts +++ b/server/core/middlewares/validators/config.ts @@ -71,6 +71,11 @@ const customConfigUpdateValidator = [ body('import.videos.torrent.enabled').isBoolean(), body('import.videoChannelSynchronization.enabled').isBoolean(), + body('import.users.enabled').isBoolean(), + + body('export.users.enabled').isBoolean(), + body('export.users.maxUserVideoQuota').exists(), + body('export.users.exportExpiration').exists(), body('trending.videos.algorithms.default').exists(), body('trending.videos.algorithms.enabled').exists(), diff --git a/server/core/middlewares/validators/index.ts b/server/core/middlewares/validators/index.ts index 5c6beb40e..26f3d8db7 100644 --- a/server/core/middlewares/validators/index.ts +++ b/server/core/middlewares/validators/index.ts @@ -21,12 +21,7 @@ export * from './server.js' export * from './sort.js' export * from './static.js' export * from './themes.js' -export * from './user-email-verification.js' -export * from './user-history.js' -export * from './user-notifications.js' -export * from './user-registrations.js' -export * from './user-subscriptions.js' -export * from './users.js' -export * from './videos/index.js' export * from './webfinger.js' +export * from './users/index.js' +export * from './videos/index.js' export * from './runners/index.js' diff --git a/server/core/middlewares/validators/shared/videos.ts b/server/core/middlewares/validators/shared/videos.ts index 0e7dfebcb..28c90a081 100644 --- a/server/core/middlewares/validators/shared/videos.ts +++ b/server/core/middlewares/validators/shared/videos.ts @@ -2,7 +2,7 @@ import { Request, Response } from 'express' import { HttpStatusCode, ServerErrorCode, UserRight, UserRightType, VideoPrivacy } from '@peertube/peertube-models' import { exists } from '@server/helpers/custom-validators/misc.js' import { loadVideo, VideoLoadType } from '@server/lib/model-loaders/index.js' -import { isAbleToUploadVideo } from '@server/lib/user.js' +import { isUserQuotaValid } from '@server/lib/user.js' import { VideoTokensManager } from '@server/lib/video-tokens-manager.js' import { authenticatePromise } from '@server/middlewares/auth.js' import { VideoChannelModel } from '@server/models/video/video-channel.js' @@ -285,7 +285,7 @@ function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: // --------------------------------------------------------------------------- async function checkUserQuota (user: MUserId, videoFileSize: number, res: Response) { - if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { + if (await isUserQuotaValid({ userId: user.id, uploadSize: videoFileSize }) === false) { res.fail({ status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, message: 'The user video quota is exceeded with this video.', diff --git a/server/core/middlewares/validators/users/index.ts b/server/core/middlewares/validators/users/index.ts new file mode 100644 index 000000000..36518a2ab --- /dev/null +++ b/server/core/middlewares/validators/users/index.ts @@ -0,0 +1,7 @@ +export * from './user-email-verification.js' +export * from './user-exports.js' +export * from './user-history.js' +export * from './user-notifications.js' +export * from './user-registrations.js' +export * from './user-subscriptions.js' +export * from './users.js' diff --git a/server/core/middlewares/validators/users/shared/index.ts b/server/core/middlewares/validators/users/shared/index.ts new file mode 100644 index 000000000..09bacf1c6 --- /dev/null +++ b/server/core/middlewares/validators/users/shared/index.ts @@ -0,0 +1 @@ +export * from './user-registrations.js' diff --git a/server/core/middlewares/validators/shared/user-registrations.ts b/server/core/middlewares/validators/users/shared/user-registrations.ts similarity index 100% rename from server/core/middlewares/validators/shared/user-registrations.ts rename to server/core/middlewares/validators/users/shared/user-registrations.ts diff --git a/server/core/middlewares/validators/user-email-verification.ts b/server/core/middlewares/validators/users/user-email-verification.ts similarity index 96% rename from server/core/middlewares/validators/user-email-verification.ts rename to server/core/middlewares/validators/users/user-email-verification.ts index 50151f642..38f5e6586 100644 --- a/server/core/middlewares/validators/user-email-verification.ts +++ b/server/core/middlewares/validators/users/user-email-verification.ts @@ -2,9 +2,9 @@ import express from 'express' import { body, param } from 'express-validator' import { toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' import { HttpStatusCode } from '@peertube/peertube-models' -import { logger } from '../../helpers/logger.js' -import { Redis } from '../../lib/redis.js' -import { areValidationErrors, checkUserEmailExist, checkUserIdExist } from './shared/index.js' +import { logger } from '../../../helpers/logger.js' +import { Redis } from '../../../lib/redis.js' +import { areValidationErrors, checkUserEmailExist, checkUserIdExist } from '../shared/index.js' import { checkRegistrationEmailExist, checkRegistrationIdExist } from './shared/user-registrations.js' const usersAskSendVerifyEmailValidator = [ diff --git a/server/core/middlewares/validators/users/user-exports.ts b/server/core/middlewares/validators/users/user-exports.ts new file mode 100644 index 000000000..e586fed2b --- /dev/null +++ b/server/core/middlewares/validators/users/user-exports.ts @@ -0,0 +1,155 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { HttpStatusCode, ServerErrorCode, UserExportRequest, UserExportState, UserRight } from '@peertube/peertube-models' +import { areValidationErrors, checkUserIdExist } from '../shared/index.js' +import { isBooleanValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' +import { UserExportModel } from '@server/models/user/user-export.js' +import { MUserExport } from '@server/types/models/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { getOriginalVideoFileTotalFromUser } from '@server/lib/user.js' + +export const userExportsListValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!ensureExportIsEnabled(res)) return + if (!await checkUserIdRight(req.params.userId, res)) return + + return next() + } +] + +export const userExportRequestValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + body('withVideoFiles') + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have withVideoFiles boolean'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!ensureExportIsEnabled(res)) return + if (!await checkUserIdRight(req.params.userId, res)) return + + // Check not already created + const exportsList = await UserExportModel.listByUser(res.locals.user) + if (exportsList.filter(e => e.state !== UserExportState.ERRORED).length !== 0) { + return res.fail({ + message: 'User has already processing or completed exports' + }) + } + + const body: UserExportRequest = req.body + + if (body.withVideoFiles) { + const quota = await getOriginalVideoFileTotalFromUser(res.locals.user) + + if (quota > CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA) { + return res.fail({ + message: 'User video quota exceeds the maximum limit set by the admin to create a user archive containing videos', + type: ServerErrorCode.MAX_USER_VIDEO_QUOTA_EXCEEDED_FOR_USER_EXPORT, + status: HttpStatusCode.FORBIDDEN_403 + }) + } + } + + return next() + } +] + +export const userExportDeleteValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + param('id') + .isInt().not().isEmpty().withMessage('Should have a valid id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!ensureExportIsEnabled(res)) return + if (!await checkUserIdRight(req.params.userId, res)) return + + const userExport = await UserExportModel.load(req.params.id + '') + if (!userExport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + + if (!checkUserExportRight(userExport, res)) return + + if (!userExport.canBeSafelyRemoved()) { + return res.fail({ + message: 'Cannot delete this user export because its state is not compatible with a deletion' + }) + } + + res.locals.userExport = userExport + + return next() + } +] + +export const userExportDownloadValidator = [ + param('filename').exists(), + + query('jwt').isJWT(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!ensureExportIsEnabled(res)) return + + const userExport = await UserExportModel.loadByFilename(req.params.filename) + if (!userExport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + + if (userExport.isJWTValid(req.query.jwt) !== true) return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + + res.locals.userExport = userExport + + return next() + } +] + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function checkUserIdRight (userId: number | string, res: express.Response) { + if (!await checkUserIdExist(userId, res)) return false + + const oauthUser = res.locals.oauth.token.User + + if (!oauthUser.hasRight(UserRight.MANAGE_USER_EXPORTS) && oauthUser.id !== res.locals.user.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage exports of another user' + }) + return false + } + + return true +} + +function checkUserExportRight (userExport: MUserExport, res: express.Response) { + if (userExport.userId !== res.locals.user.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Export is not associated to this user' + }) + return false + } + + return true +} + +function ensureExportIsEnabled (res: express.Response) { + if (CONFIG.EXPORT.USERS.ENABLED !== true) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'User export is disabled on this instance' + }) + + return false + } + + return true +} diff --git a/server/core/middlewares/validators/user-history.ts b/server/core/middlewares/validators/users/user-history.ts similarity index 87% rename from server/core/middlewares/validators/user-history.ts rename to server/core/middlewares/validators/users/user-history.ts index 8415dab2a..51b6c5464 100644 --- a/server/core/middlewares/validators/user-history.ts +++ b/server/core/middlewares/validators/users/user-history.ts @@ -1,7 +1,7 @@ import express from 'express' import { body, param, query } from 'express-validator' -import { exists, isDateValid, isIdValid } from '../../helpers/custom-validators/misc.js' -import { areValidationErrors } from './shared/index.js' +import { exists, isDateValid, isIdValid } from '../../../helpers/custom-validators/misc.js' +import { areValidationErrors } from '../shared/index.js' const userHistoryListValidator = [ query('search') diff --git a/server/core/middlewares/validators/users/user-import.ts b/server/core/middlewares/validators/users/user-import.ts new file mode 100644 index 000000000..bec7db019 --- /dev/null +++ b/server/core/middlewares/validators/users/user-import.ts @@ -0,0 +1,129 @@ +import express from 'express' +import { body, header, param } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload.js' +import { uploadx } from '@server/lib/uploadx.js' +import { Metadata as UploadXMetadata } from '@uploadx/core' +import { logger } from '../../../helpers/logger.js' +import { areValidationErrors, checkUserIdExist } from '../shared/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { HttpStatusCode, ServerErrorCode, UserImportState, UserRight } from '@peertube/peertube-models' +import { isUserQuotaValid } from '@server/lib/user.js' +import { UserImportModel } from '@server/models/user/user-import.js' + +export const userImportRequestResumableValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body: express.CustomUploadXFile = req.body + const file = { ...body, path: getResumableUploadPath(body.name), filename: body.metadata.filename } + const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) + + if (!await checkUserIdRight(req.params.userId, res)) return cleanup() + + if (CONFIG.IMPORT.USERS.ENABLED !== true) { + res.fail({ + message: 'User import is not enabled by the administrator', + status: HttpStatusCode.BAD_REQUEST_400 + }) + + return cleanup() + } + + res.locals.importUserFileResumable = { ...file, originalname: file.filename } + + return next() + } +] + +export const userImportRequestResumableInitValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + body('filename') + .exists(), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking userImportRequestResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers + }) + + if (areValidationErrors(req, res, { omitLog: true })) return + + if (CONFIG.IMPORT.USERS.ENABLED !== true) { + return res.fail({ + message: 'User import is not enabled by the administrator', + status: HttpStatusCode.BAD_REQUEST_400 + }) + } + + if (req.body.filename.endsWith('.zip') !== true) { + return res.fail({ + message: 'User import file must be a zip', + status: HttpStatusCode.BAD_REQUEST_400 + }) + } + + if (!await checkUserIdRight(req.params.userId, res)) return + + const user = res.locals.user + if (await isUserQuotaValid({ userId: user.id, uploadSize: +req.headers['x-upload-content-length'] }) === false) { + return res.fail({ + message: 'User video quota is exceeded with this import', + status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, + type: ServerErrorCode.QUOTA_REACHED + }) + } + + const userImport = await UserImportModel.loadLatestByUserId(user.id) + if (userImport && userImport.state !== UserImportState.ERRORED && userImport.state !== UserImportState.COMPLETED) { + return res.fail({ + message: 'An import is already being processed', + status: HttpStatusCode.BAD_REQUEST_400 + }) + } + + return next() + } +] + +export const getLatestImportStatusValidator = [ + param('userId') + .isInt().not().isEmpty().withMessage('Should have a valid userId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!await checkUserIdRight(req.params.userId, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function checkUserIdRight (userId: number | string, res: express.Response) { + if (!await checkUserIdExist(userId, res)) return false + + const oauthUser = res.locals.oauth.token.User + + if (!oauthUser.hasRight(UserRight.MANAGE_USER_IMPORTS) && oauthUser.id !== res.locals.user.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage imports of another user' + }) + return false + } + + return true +} diff --git a/server/core/middlewares/validators/user-notifications.ts b/server/core/middlewares/validators/users/user-notifications.ts similarity index 88% rename from server/core/middlewares/validators/user-notifications.ts rename to server/core/middlewares/validators/users/user-notifications.ts index 16bbc6693..e254e7cfb 100644 --- a/server/core/middlewares/validators/user-notifications.ts +++ b/server/core/middlewares/validators/users/user-notifications.ts @@ -1,8 +1,8 @@ import express from 'express' import { body, query } from 'express-validator' -import { isNotEmptyIntArray, toBooleanOrNull } from '../../helpers/custom-validators/misc.js' -import { isUserNotificationSettingValid } from '../../helpers/custom-validators/user-notifications.js' -import { areValidationErrors } from './shared/index.js' +import { isNotEmptyIntArray, toBooleanOrNull } from '../../../helpers/custom-validators/misc.js' +import { isUserNotificationSettingValid } from '../../../helpers/custom-validators/user-notifications.js' +import { areValidationErrors } from '../shared/index.js' const listUserNotificationsValidator = [ query('unread') diff --git a/server/core/middlewares/validators/user-registrations.ts b/server/core/middlewares/validators/users/user-registrations.ts similarity index 96% rename from server/core/middlewares/validators/user-registrations.ts rename to server/core/middlewares/validators/users/user-registrations.ts index 392fe52a6..c9d06ba99 100644 --- a/server/core/middlewares/validators/user-registrations.ts +++ b/server/core/middlewares/validators/users/user-registrations.ts @@ -5,11 +5,11 @@ import { isRegistrationModerationResponseValid, isRegistrationReasonValid } from import { CONFIG } from '@server/initializers/config.js' import { Hooks } from '@server/lib/plugins/hooks.js' import { HttpStatusCode, UserRegister, UserRegistrationRequest, UserRegistrationState } from '@peertube/peertube-models' -import { isUserDisplayNameValid, isUserPasswordValid, isUserUsernameValid } from '../../helpers/custom-validators/users.js' -import { isVideoChannelDisplayNameValid, isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels.js' -import { isSignupAllowed, isSignupAllowedForCurrentIP, SignupMode } from '../../lib/signup.js' -import { ActorModel } from '../../models/actor/actor.js' -import { areValidationErrors, checkUserNameOrEmailDoNotAlreadyExist } from './shared/index.js' +import { isUserDisplayNameValid, isUserPasswordValid, isUserUsernameValid } from '../../../helpers/custom-validators/users.js' +import { isVideoChannelDisplayNameValid, isVideoChannelUsernameValid } from '../../../helpers/custom-validators/video-channels.js' +import { isSignupAllowed, isSignupAllowedForCurrentIP, SignupMode } from '../../../lib/signup.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { areValidationErrors, checkUserNameOrEmailDoNotAlreadyExist } from '../shared/index.js' import { checkRegistrationHandlesDoNotAlreadyExist, checkRegistrationIdExist } from './shared/user-registrations.js' const usersDirectRegistrationValidator = usersCommonRegistrationValidatorFactory() diff --git a/server/core/middlewares/validators/user-subscriptions.ts b/server/core/middlewares/validators/users/user-subscriptions.ts similarity index 92% rename from server/core/middlewares/validators/user-subscriptions.ts rename to server/core/middlewares/validators/users/user-subscriptions.ts index 78bea840f..181935ea2 100644 --- a/server/core/middlewares/validators/user-subscriptions.ts +++ b/server/core/middlewares/validators/users/user-subscriptions.ts @@ -2,10 +2,10 @@ import express from 'express' import { body, param, query } from 'express-validator' import { arrayify } from '@peertube/peertube-core-utils' import { FollowState, HttpStatusCode } from '@peertube/peertube-models' -import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor.js' -import { WEBSERVER } from '../../initializers/constants.js' -import { ActorFollowModel } from '../../models/actor/actor-follow.js' -import { areValidationErrors } from './shared/index.js' +import { areValidActorHandles, isValidActorHandle } from '../../../helpers/custom-validators/activitypub/actor.js' +import { WEBSERVER } from '../../../initializers/constants.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { areValidationErrors } from '../shared/index.js' const userSubscriptionListValidator = [ query('search') diff --git a/server/core/middlewares/validators/users.ts b/server/core/middlewares/validators/users/users.ts similarity index 96% rename from server/core/middlewares/validators/users.ts rename to server/core/middlewares/validators/users/users.ts index 010ae496d..fa047597f 100644 --- a/server/core/middlewares/validators/users.ts +++ b/server/core/middlewares/validators/users/users.ts @@ -2,8 +2,8 @@ import express from 'express' import { body, param, query } from 'express-validator' import { forceNumber } from '@peertube/peertube-core-utils' import { HttpStatusCode, UserRight, UserRole } from '@peertube/peertube-models' -import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc.js' -import { isThemeNameValid } from '../../helpers/custom-validators/plugins.js' +import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc.js' +import { isThemeNameValid } from '../../../helpers/custom-validators/plugins.js' import { isUserAdminFlagsValid, isUserAutoPlayNextVideoValid, @@ -23,12 +23,12 @@ import { isUserVideoQuotaDailyValid, isUserVideoQuotaValid, isUserVideosHistoryEnabledValid -} from '../../helpers/custom-validators/users.js' -import { isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels.js' -import { logger } from '../../helpers/logger.js' -import { isThemeRegistered } from '../../lib/plugins/theme-utils.js' -import { Redis } from '../../lib/redis.js' -import { ActorModel } from '../../models/actor/actor.js' +} from '../../../helpers/custom-validators/users.js' +import { isVideoChannelUsernameValid } from '../../../helpers/custom-validators/video-channels.js' +import { logger } from '../../../helpers/logger.js' +import { isThemeRegistered } from '../../../lib/plugins/theme-utils.js' +import { Redis } from '../../../lib/redis.js' +import { ActorModel } from '../../../models/actor/actor.js' import { areValidationErrors, checkUserEmailExist, @@ -37,7 +37,7 @@ import { doesVideoChannelIdExist, doesVideoExist, isValidVideoIdParam -} from './shared/index.js' +} from '../shared/index.js' const usersListValidator = [ query('blocked') diff --git a/server/core/middlewares/validators/videos/shared/video-validators.ts b/server/core/middlewares/validators/videos/shared/video-validators.ts index a3248463a..1ac7308fd 100644 --- a/server/core/middlewares/validators/videos/shared/video-validators.ts +++ b/server/core/middlewares/validators/videos/shared/video-validators.ts @@ -19,8 +19,7 @@ export async function commonVideoFileChecks (options: { if (!isVideoFileMimeTypeValid(files)) { res.fail({ status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, - message: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + message: `This file is not supported. Please, make sure it is of the following type: ${CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')}` }) return false } diff --git a/server/core/middlewares/validators/videos/video-rates.ts b/server/core/middlewares/validators/videos/video-rates.ts index f7b784fe3..6adbf1a6e 100644 --- a/server/core/middlewares/validators/videos/video-rates.ts +++ b/server/core/middlewares/validators/videos/video-rates.ts @@ -13,6 +13,7 @@ const videoUpdateRateValidator = [ body('rating') .custom(isVideoRatingTypeValid), + isValidVideoPasswordHeader(), async (req: express.Request, res: express.Response, next: express.NextFunction) => { diff --git a/server/core/models/account/account-blocklist.ts b/server/core/models/account/account-blocklist.ts index fa7fa8021..bc088a4e5 100644 --- a/server/core/models/account/account-blocklist.ts +++ b/server/core/models/account/account-blocklist.ts @@ -8,6 +8,7 @@ import { ActorModel } from '../actor/actor.js' import { ServerModel } from '../server/server.js' import { createSafeIn, getSort, searchAttribute } from '../shared/index.js' import { AccountModel } from './account.js' +import { WEBSERVER } from '@server/initializers/constants.js' @Table({ tableName: 'accountBlocklist', @@ -180,7 +181,7 @@ export class AccountBlocklistModel extends Model entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`)) + .then(entries => { + return entries.map(e => { + const host = e.BlockedAccount.Actor.Server?.host ?? WEBSERVER.HOST + + return `${e.BlockedAccount.Actor.preferredUsername}@${host}` + }) + }) } static getBlockStatus (byAccountIds: number[], handles: string[]): Promise<{ name: string, host: string, accountId: number }[]> { diff --git a/server/core/models/account/account-video-rate.ts b/server/core/models/account/account-video-rate.ts index 0d099ff9f..3f9c93808 100644 --- a/server/core/models/account/account-video-rate.ts +++ b/server/core/models/account/account-video-rate.ts @@ -4,7 +4,8 @@ import { MAccountVideoRate, MAccountVideoRateAccountUrl, MAccountVideoRateAccountVideo, - MAccountVideoRateFormattable + MAccountVideoRateFormattable, + MAccountVideoRateVideoUrl } from '@server/types/models/index.js' import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' @@ -113,6 +114,59 @@ export class AccountVideoRateModel extends Model { + const options: FindOptions = { + where: { + videoId, + type: rateType + }, + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'url', 'followersUrl', 'preferredUsername' ], + model: ActorModel.unscoped(), + required: true, + where: { + [Op.and]: [ + ActorModel.wherePreferredUsername(accountName), + { serverId: null } + ] + } + } + ] + }, + { + model: VideoModel.unscoped(), + required: true + } + ] + } + if (t) options.transaction = t + + return AccountVideoRateModel.findOne(options) + } + + static loadByUrl (url: string, transaction: Transaction) { + const options: FindOptions = { + where: { + url + } + } + if (transaction) options.transaction = transaction + + return AccountVideoRateModel.findOne(options) + } + + // --------------------------------------------------------------------------- + static listByAccountForApi (options: { start: number count: number @@ -168,57 +222,6 @@ export class AccountVideoRateModel extends Model rows.map(r => r.url)) } - static loadLocalAndPopulateVideo ( - rateType: VideoRateType, - accountName: string, - videoId: number, - t?: Transaction - ): Promise { - const options: FindOptions = { - where: { - videoId, - type: rateType - }, - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'url', 'followersUrl', 'preferredUsername' ], - model: ActorModel.unscoped(), - required: true, - where: { - [Op.and]: [ - ActorModel.wherePreferredUsername(accountName), - { serverId: null } - ] - } - } - ] - }, - { - model: VideoModel.unscoped(), - required: true - } - ] - } - if (t) options.transaction = t - - return AccountVideoRateModel.findOne(options) - } - - static loadByUrl (url: string, transaction: Transaction) { - const options: FindOptions = { - where: { - url - } - } - if (transaction) options.transaction = transaction - - return AccountVideoRateModel.findOne(options) - } - static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) { const query = { offset: start, @@ -250,6 +253,26 @@ export class AccountVideoRateModel extends Model ({ total, data })) } + static listRatesOfAccountId (accountId: number, rateType: VideoRateType): Promise { + const query = { + where: { + accountId, + type: rateType + }, + include: [ + { + attributes: [ 'url' ], + model: VideoModel, + required: true + } + ] + } + + return AccountVideoRateModel.findAll(query) + } + + // --------------------------------------------------------------------------- + toFormattedJSON (this: MAccountVideoRateFormattable): AccountVideoRate { return { video: this.Video.toFormattedJSON(), diff --git a/server/core/models/actor/actor-follow.ts b/server/core/models/actor/actor-follow.ts index 268edb5b4..38ae4b764 100644 --- a/server/core/models/actor/actor-follow.ts +++ b/server/core/models/actor/actor-follow.ts @@ -496,26 +496,98 @@ export class ActorFollowModel extends Model ({ total, data: data.map(d => d.selectionUrl) })) } static listAcceptedFollowerSharedInboxUrls (actorIds: number[], t: Transaction) { - return ActorFollowModel.createListAcceptedFollowForApiQuery( - 'followers', + return ActorFollowModel.createListAcceptedFollowForApiQuery({ + type: 'followers', actorIds, t, - undefined, - undefined, - 'sharedInboxUrl', - true - ) + columnUrl: 'sharedInboxUrl', + distinct: true + }).then(({ data, total }) => ({ total, data: data.map(d => d.selectionUrl) })) } - static listAcceptedFollowingUrlsForApi (actorIds: number[], t: Transaction, start?: number, count?: number) { - return ActorFollowModel.createListAcceptedFollowForApiQuery('following', actorIds, t, start, count) + static listAcceptedFollowersForExport (targetActorId: number) { + const query = { + where: { + state: 'accepted', + targetActorId + }, + include: [ + { + attributes: [ 'preferredUsername', 'url' ], + model: ActorModel.unscoped(), + required: true, + as: 'ActorFollower', + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + } + ] + } + ] + } + + return ActorFollowModel.findAll(query) + .then(data => { + return data.map(f => ({ + createdAt: f.createdAt, + followerHandle: f.ActorFollower.getFullIdentifier(), + followerUrl: f.ActorFollower.url + })) + }) } + // --------------------------------------------------------------------------- + + static listAcceptedFollowingUrlsForApi (actorIds: number[], t: Transaction, start?: number, count?: number) { + return ActorFollowModel.createListAcceptedFollowForApiQuery({ type: 'following', actorIds, t, start, count }) + .then(({ data, total }) => ({ total, data: data.map(d => d.selectionUrl) })) + } + + static listAcceptedFollowingForExport (actorId: number) { + const query = { + where: { + state: 'accepted', + actorId + }, + include: [ + { + attributes: [ 'preferredUsername', 'url' ], + model: ActorModel.unscoped(), + required: true, + as: 'ActorFollowing', + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + } + ] + } + ] + } + + return ActorFollowModel.findAll(query) + .then(data => { + return data.map(f => ({ + createdAt: f.createdAt, + followingHandle: f.ActorFollowing.getFullIdentifier(), + followingUrl: f.ActorFollowing.url + })) + }) + } + + // --------------------------------------------------------------------------- + static async getStats () { const serverActor = await getServerActor() @@ -577,15 +649,21 @@ export class ActorFollowModel extends Model[] = [] @@ -622,12 +704,14 @@ export class ActorFollowModel extends Model f.selectionUrl) + const [ followers, resDataTotal ] = await Promise.all(tasks) return { - data: urls, - total: dataTotal ? parseInt(dataTotal.total, 10) : 0 + data: followers.map(f => ({ selectionUrl: f.selectionUrl, createdAt: f.createdAt })) as { selectionUrl: string, createdAt: string }[], + + total: selectTotal + ? parseInt(resDataTotal?.dataTotal?.[0]?.total || 0, 10) + : undefined } } diff --git a/server/core/models/actor/actor.ts b/server/core/models/actor/actor.ts index adb009968..64d8cf5c6 100644 --- a/server/core/models/actor/actor.ts +++ b/server/core/models/actor/actor.ts @@ -63,6 +63,7 @@ import { VideoChannelModel } from '../video/video-channel.js' import { VideoModel } from '../video/video.js' import { ActorFollowModel } from './actor-follow.js' import { ActorImageModel } from './actor-image.js' +import maxBy from 'lodash-es/maxBy.js' enum ScopeNames { FULL = 'FULL' @@ -662,6 +663,10 @@ export class ActorModel extends Model>> { return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername } + getFullIdentifier (this: MActorHost) { + return `${this.preferredUsername}@${this.getHost()}` + } + getHost (this: MActorHostOnly) { return this.Server ? this.Server.host : WEBSERVER.HOST } @@ -678,6 +683,16 @@ export class ActorModel extends Model>> { return Array.isArray(images) && images.length !== 0 } + getMaxQualityImage (type: ActorImageType_Type) { + if (!this.hasImage(type)) return undefined + + const images = type === ActorImageType.AVATAR + ? this.Avatars + : this.Banners + + return maxBy(images, 'height') + } + isOutdated () { if (this.isOwned()) return false diff --git a/server/core/models/user/user-export.ts b/server/core/models/user/user-export.ts new file mode 100644 index 000000000..68bc3cd38 --- /dev/null +++ b/server/core/models/user/user-export.ts @@ -0,0 +1,213 @@ +import { FindOptions, Op } from 'sequelize' +import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MUserAccountId, MUserExport } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { UserModel } from './user.js' +import { getSort } from '../shared/sort.js' +import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { remove } from 'fs-extra/esm' +import { getFSUserExportFilePath } from '@server/lib/paths.js' +import { + JWT_TOKEN_USER_EXPORT_FILE_LIFETIME, + STATIC_DOWNLOAD_PATHS, + USER_EXPORT_STATES, + WEBSERVER +} from '@server/initializers/constants.js' +import { join } from 'path' +import jwt from 'jsonwebtoken' +import { CONFIG } from '@server/initializers/config.js' +import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js' + +@Table({ + tableName: 'userExport', + indexes: [ + { + fields: [ 'userId' ] + }, + { + fields: [ 'filename' ], + unique: true + } + ] +}) +export class UserExportModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Column + filename: string + + @AllowNull(false) + @Column + withVideoFiles: boolean + + @AllowNull(false) + @Column + state: UserExportStateType + + @AllowNull(true) + @Column(DataType.TEXT) + error: string + + @AllowNull(true) + @Column + size: number + + @AllowNull(false) + @Column + storage: FileStorageType + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + User: Awaited + + @BeforeDestroy + static removeFile (instance: UserExportModel) { + logger.info('Removing user export file %s.', instance.filename) + + if (instance.storage === FileStorage.FILE_SYSTEM) { + remove(getFSUserExportFilePath(instance)) + .catch(err => logger.error('Cannot delete user export archive %s from filesystem.', instance.filename, { err })) + } else { + removeUserExportObjectStorage(instance) + .catch(err => logger.error('Cannot delete user export archive %s from object storage.', instance.filename, { err })) + } + + return undefined + } + + static listByUser (user: MUserAccountId) { + const query: FindOptions = { + where: { + userId: user.id + } + } + + return UserExportModel.findAll(query) + } + + static listExpired (expirationTimeMS: number) { + const query: FindOptions = { + where: { + createdAt: { + [Op.lt]: new Date(new Date().getTime() + expirationTimeMS) + } + } + } + + return UserExportModel.findAll(query) + } + + static listForApi (options: { + user: MUserAccountId + start: number + count: number + }) { + const { count, start, user } = options + + const query: FindOptions = { + offset: start, + limit: count, + order: getSort('createdAt'), + where: { + userId: user.id + } + } + + return Promise.all([ + UserExportModel.count(query), + UserExportModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static load (id: number | string) { + return UserExportModel.findByPk(id) + } + + static loadByFilename (filename: string) { + return UserExportModel.findOne({ where: { filename } }) + } + + // --------------------------------------------------------------------------- + + generateAndSetFilename () { + if (!this.userId) throw new Error('Cannot generate filename without userId') + if (!this.createdAt) throw new Error('Cannot generate filename without createdAt') + + this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip` + } + + canBeSafelyRemoved () { + const supportedStates = new Set([ UserExportState.COMPLETED, UserExportState.ERRORED, UserExportState.PENDING ]) + + return supportedStates.has(this.state) + } + + generateJWT () { + return jwt.sign( + { + userExportId: this.id + }, + CONFIG.SECRETS.PEERTUBE, + { + expiresIn: JWT_TOKEN_USER_EXPORT_FILE_LIFETIME, + audience: this.filename, + issuer: WEBSERVER.URL + } + ) + } + + isJWTValid (jwtToken: string) { + try { + const payload = jwt.verify(jwtToken, CONFIG.SECRETS.PEERTUBE, { + audience: this.filename, + issuer: WEBSERVER.URL + }) + + if ((payload as any).userExportId !== this.id) return false + + return true + } catch { + return false + } + } + + getFileDownloadUrl () { + if (this.state !== UserExportState.COMPLETED) return null + + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.USER_EXPORT, this.filename) + '?jwt=' + this.generateJWT() + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MUserExport): UserExport { + return { + id: this.id, + + state: { + id: this.state, + label: USER_EXPORT_STATES[this.state] + }, + + size: this.size, + + privateDownloadUrl: this.getFileDownloadUrl(), + createdAt: this.createdAt.toISOString(), + expiresOn: new Date(this.createdAt.getTime() + CONFIG.EXPORT.USERS.EXPORT_EXPIRATION).toISOString() + } + } + +} diff --git a/server/core/models/user/user-import.ts b/server/core/models/user/user-import.ts new file mode 100644 index 000000000..55c0bc86e --- /dev/null +++ b/server/core/models/user/user-import.ts @@ -0,0 +1,88 @@ +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MUserImport } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { UserModel } from './user.js' +import type { UserImportResultSummary, UserImportStateType } from '@peertube/peertube-models' +import { getSort } from '../shared/sort.js' +import { USER_IMPORT_STATES } from '@server/initializers/constants.js' + +@Table({ + tableName: 'userImport', + indexes: [ + { + fields: [ 'userId' ] + }, + { + fields: [ 'filename' ], + unique: true + } + ] +}) +export class UserImportModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Column + filename: string + + @AllowNull(false) + @Column + state: UserImportStateType + + @AllowNull(true) + @Column(DataType.TEXT) + error: string + + @AllowNull(true) + @Column(DataType.JSONB) + resultSummary: UserImportResultSummary + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + User: Awaited + + static load (id: number | string) { + return UserImportModel.findByPk(id) + } + + static loadLatestByUserId (userId: number) { + return UserImportModel.findOne({ + where: { + userId + }, + order: getSort('-createdAt') + }) + } + + // --------------------------------------------------------------------------- + + generateAndSetFilename () { + if (!this.userId) throw new Error('Cannot generate filename without userId') + if (!this.createdAt) throw new Error('Cannot generate filename without createdAt') + + this.filename = `user-import-${this.userId}-${this.createdAt.toISOString()}.zip` + } + + toFormattedJSON () { + return { + id: this.id, + state: { + id: this.state, + label: USER_IMPORT_STATES[this.state] + }, + createdAt: this.createdAt.toISOString() + } + } +} diff --git a/server/core/models/user/user-notification-setting.ts b/server/core/models/user/user-notification-setting.ts index 8b59fbe70..8d51a1880 100644 --- a/server/core/models/user/user-notification-setting.ts +++ b/server/core/models/user/user-notification-setting.ts @@ -208,6 +208,16 @@ export class UserNotificationSettingModel extends Model>> { }) OAuthTokens: Awaited[] + @HasMany(() => UserExportModel, { + foreignKey: 'userId', + onDelete: 'cascade', + hooks: true + }) + UserExports: Awaited[] + // Used if we already set an encrypted password in user model skipPasswordEncryption = false @@ -724,6 +730,23 @@ export class UserModel extends Model>> { return UserModel.findOne(query) } + static loadByAccountId (accountId: number): Promise { + const query = { + include: [ + { + required: true, + attributes: [ 'id' ], + model: AccountModel.unscoped(), + where: { + id: accountId + } + } + ] + } + + return UserModel.findOne(query) + } + static loadByAccountActorId (accountActorId: number): Promise { const query = { include: [ @@ -780,17 +803,18 @@ export class UserModel extends Model>> { } static generateUserQuotaBaseSQL (options: { - whereUserId: '$userId' | '"UserModel"."id"' - withSelect: boolean daily: boolean + whereUserId: '$userId' | '"UserModel"."id"' }) { - const andWhere = options.daily === true + const { daily, whereUserId } = options + + const andWhere = daily === true ? 'AND "video"."createdAt" > now() - interval \'24 hours\'' : '' const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + - `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` + `WHERE "account"."userId" = ${whereUserId} ${andWhere}` const webVideoFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" AND "video"."isLive" IS FALSE ' + @@ -808,18 +832,23 @@ export class UserModel extends Model>> { ') t2' } - static getTotalRawQuery (query: string, userId: number) { - const options = { + static async getUserQuota (options: { + userId: number + daily: boolean + }) { + const { daily, userId } = options + + const sql = this.generateUserQuotaBaseSQL({ daily, whereUserId: '$userId' }) + + const queryOptions = { bind: { userId }, type: QueryTypes.SELECT as QueryTypes.SELECT } - return UserModel.sequelize.query<{ total: string }>(query, options) - .then(([ { total } ]) => { - if (total === null) return 0 + const [ { total } ] = await UserModel.sequelize.query<{ total: string }>(sql, queryOptions) + if (!total) return 0 - return parseInt(total, 10) - }) + return parseInt(total, 10) } static async getStats () { diff --git a/server/core/models/video/formatter/video-activity-pub-format.ts b/server/core/models/video/formatter/video-activity-pub-format.ts index 691948a47..bfa28cbca 100644 --- a/server/core/models/video/formatter/video-activity-pub-format.ts +++ b/server/core/models/video/formatter/video-activity-pub-format.ts @@ -8,9 +8,7 @@ import { ActivityPubStoryboard, ActivityTagObject, ActivityTrackerUrlObject, - ActivityUrlObject, - ActivityVideoUrlObject, - VideoObject + ActivityUrlObject, VideoObject } from '@peertube/peertube-models' import { WEBSERVER } from '../../../initializers/constants.js' import { @@ -21,10 +19,8 @@ import { getLocalVideoSharesActivityPubUrl } from '../../../lib/activitypub/url.js' import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models/index.js' -import { VideoCaptionModel } from '../video-caption.js' import { sortByResolutionDesc } from './shared/index.js' import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format.js' -import { getVideoFileMimeType } from '@server/lib/video-file.js' export function videoModelToActivityPubObject (video: MVideoAP): VideoObject { const language = video.language @@ -181,21 +177,12 @@ function buildVideoFileUrls (options: { .sort(sortByResolutionDesc) for (const file of sortedFiles) { - // FIXME: Replace false by file.isAudio(), federation breaking change correctly handled in 6.0 - const mimeType = getVideoFileMimeType(file.extname, false) + const fileAP = file.toActivityPubObject(video) + urls.push(fileAP) urls.push({ type: 'Link', - mediaType: mimeType, - href: file.getFileUrl(video), - height: file.resolution, - size: file.size, - fps: file.fps - } as ActivityVideoUrlObject) - - urls.push({ - type: 'Link', - rel: [ 'metadata', mimeType ], + rel: [ 'metadata', fileAP.mediaType ], mediaType: 'application/json' as 'application/json', href: getLocalVideoFileMetadataUrl(video, file), height: file.resolution, @@ -282,22 +269,12 @@ function buildTags (video: MVideoAP) { function buildIcon (video: MVideoAP): ActivityIconObject[] { return [ video.getMiniature(), video.getPreview() ] - .map(i => ({ - type: 'Image', - url: i.getOriginFileUrl(video), - mediaType: 'image/jpeg', - width: i.width, - height: i.height - })) + .map(i => i.toActivityPubObject(video)) } function buildSubtitleLanguage (video: MVideoAP) { if (!isArray(video.VideoCaptions)) return [] return video.VideoCaptions - .map(caption => ({ - identifier: caption.language, - name: VideoCaptionModel.getLanguageLabel(caption.language), - url: caption.getFileUrl(video) - })) + .map(caption => caption.toActivityPubObject(video)) } diff --git a/server/core/models/video/sql/video/video-model-get-query-builder.ts b/server/core/models/video/sql/video/video-model-get-query-builder.ts index 1d55e3e93..885ee4a34 100644 --- a/server/core/models/video/sql/video/video-model-get-query-builder.ts +++ b/server/core/models/video/sql/video/video-model-get-query-builder.ts @@ -15,6 +15,7 @@ export type GetType = 'api' | 'full' | 'account-blacklist-files' | + 'account' | 'all-files' | 'thumbnails' | 'thumbnails-blacklist' | @@ -100,7 +101,7 @@ export class VideosModelGetQuerySubBuilder extends AbstractVideoQueryBuilder { private static readonly scheduleUpdateInclude = new Set([ 'api', 'full' ]) private static readonly tagsInclude = new Set([ 'api', 'full' ]) private static readonly userHistoryInclude = new Set([ 'api', 'full' ]) - private static readonly accountInclude = new Set([ 'api', 'full', 'account-blacklist-files' ]) + private static readonly accountInclude = new Set([ 'api', 'full', 'account', 'account-blacklist-files' ]) private static readonly ownerUserInclude = new Set([ 'blacklist-rights' ]) private static readonly blacklistedInclude = new Set([ diff --git a/server/core/models/video/thumbnail.ts b/server/core/models/video/thumbnail.ts index 4791b8b75..8e06fed96 100644 --- a/server/core/models/video/thumbnail.ts +++ b/server/core/models/video/thumbnail.ts @@ -1,7 +1,7 @@ -import { ThumbnailType, type ThumbnailType_Type } from '@peertube/peertube-models' +import { ActivityIconObject, ThumbnailType, type ThumbnailType_Type } from '@peertube/peertube-models' import { AttributesOnly } from '@peertube/peertube-typescript-utils' import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' -import { MThumbnail, MThumbnailVideo, MVideo } from '@server/types/models/index.js' +import { MThumbnail, MThumbnailVideo, MVideo, MVideoPlaylist } from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' import { join } from 'path' import { @@ -168,10 +168,10 @@ export class ThumbnailModel extends Model return join(directory, filename) } - getOriginFileUrl (video: MVideo) { + getOriginFileUrl (videoOrPlaylist: MVideo | MVideoPlaylist) { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename - if (video.isOwned()) return WEBSERVER.URL + staticPath + if (videoOrPlaylist.isOwned()) return WEBSERVER.URL + staticPath return this.fileUrl } @@ -205,4 +205,16 @@ export class ThumbnailModel extends Model isOwned () { return !this.fileUrl } + + // --------------------------------------------------------------------------- + + toActivityPubObject (this: MThumbnail, video: MVideo): ActivityIconObject { + return { + type: 'Image', + url: this.getOriginFileUrl(video), + mediaType: 'image/jpeg', + width: this.width, + height: this.height + } + } } diff --git a/server/core/models/video/video-caption.ts b/server/core/models/video/video-caption.ts index 312f23a30..0d5da3e31 100644 --- a/server/core/models/video/video-caption.ts +++ b/server/core/models/video/video-caption.ts @@ -15,7 +15,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { VideoCaption } from '@peertube/peertube-models' +import { ActivityIdentifierObject, VideoCaption } from '@peertube/peertube-models' import { MVideo, MVideoCaption, @@ -220,6 +220,8 @@ export class VideoCaptionModel extends Model ({ total, data })) } - static listAllByAccount (accountId: number): Promise { + static listAllByAccount (accountId: number): Promise { const query = { limit: CONFIG.VIDEO_CHANNELS.MAX_PER_USER, include: [ diff --git a/server/core/models/video/video-comment.ts b/server/core/models/video/video-comment.ts index 052c53a3d..44307dcc6 100644 --- a/server/core/models/video/video-comment.ts +++ b/server/core/models/video/video-comment.ts @@ -25,6 +25,7 @@ import { MComment, MCommentAdminFormattable, MCommentAP, + MCommentExport, MCommentFormattable, MCommentId, MCommentOwner, @@ -461,6 +462,31 @@ export class VideoCommentModel extends Model() } + static listForExport (ofAccountId: number): Promise { + const query = { + attributes: [ 'url', 'text', 'createdAt' ], + where: { + accountId: ofAccountId, + deletedAt: null + }, + include: [ + { + attributes: [ 'url' ], + required: true, + model: VideoModel.unscoped() + }, + { + attributes: [ 'url' ], + required: false, + model: VideoCommentModel, + as: 'InReplyToVideoComment' + } + ] + } + + return VideoCommentModel.findAll(query) + } + static async getStats () { const totalLocalVideoComments = await VideoCommentModel.count({ include: [ diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts index ee6e11ca3..e1532e73f 100644 --- a/server/core/models/video/video-file.ts +++ b/server/core/models/video/video-file.ts @@ -1,4 +1,4 @@ -import { VideoResolution, VideoStorage, type VideoStorageType } from '@peertube/peertube-models' +import { ActivityVideoUrlObject, VideoResolution, FileStorage, type FileStorageType } from '@peertube/peertube-models' import { AttributesOnly } from '@peertube/peertube-typescript-utils' import { logger } from '@server/helpers/logger.js' import { extractVideo } from '@server/helpers/video.js' @@ -54,6 +54,7 @@ import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' import { doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js' import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js' import { VideoModel } from './video.js' +import { getVideoFileMimeType } from '@server/lib/video-file.js' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO', @@ -223,9 +224,9 @@ export class VideoFileModel extends Model videoId: number @AllowNull(false) - @Default(VideoStorage.FILE_SYSTEM) + @Default(FileStorage.FILE_SYSTEM) @Column - storage: VideoStorageType + storage: FileStorageType @BelongsTo(() => VideoModel, { foreignKey: { @@ -286,7 +287,7 @@ export class VideoFileModel extends Model static async doesOwnedWebVideoFileExist (filename: string) { const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + - `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` + `WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1` return doesExist(this.sequelize, query, { filename }) } @@ -538,7 +539,7 @@ export class VideoFileModel extends Model getFileUrl (video: MVideo) { if (video.isOwned()) { - if (this.storage === VideoStorage.OBJECT_STORAGE) { + if (this.storage === FileStorage.OBJECT_STORAGE) { return this.getObjectStorageUrl(video) } @@ -632,4 +633,19 @@ export class VideoFileModel extends Model return Object.assign(this, { Video: videoOrPlaylist }) } + + // --------------------------------------------------------------------------- + + toActivityPubObject (this: MVideoFile, video: MVideo): ActivityVideoUrlObject { + const mimeType = getVideoFileMimeType(this.extname, false) + + return { + type: 'Link', + mediaType: mimeType as ActivityVideoUrlObject['mediaType'], + href: this.getFileUrl(video), + height: this.resolution, + size: this.size, + fps: this.fps + } + } } diff --git a/server/core/models/video/video-live.ts b/server/core/models/video/video-live.ts index fefaa053a..10c0d8594 100644 --- a/server/core/models/video/video-live.ts +++ b/server/core/models/video/video-live.ts @@ -2,7 +2,7 @@ import { LiveVideo, VideoState, type LiveVideoLatencyModeType } from '@peertube/ import { AttributesOnly } from '@peertube/peertube-typescript-utils' import { CONFIG } from '@server/initializers/config.js' import { WEBSERVER } from '@server/initializers/constants.js' -import { MVideoLive, MVideoLiveVideoWithSetting } from '@server/types/models/index.js' +import { MVideoLive, MVideoLiveVideoWithSetting, MVideoLiveWithSetting } from '@server/types/models/index.js' import { Transaction } from 'sequelize' import { AllowNull, @@ -149,6 +149,22 @@ export class VideoLiveModel extends Model return VideoLiveModel.findOne(query) } + static loadByVideoIdWithSettings (videoId: number) { + const query = { + where: { + videoId + }, + include: [ + { + model: VideoLiveReplaySettingModel.unscoped(), + required: false + } + ] + } + + return VideoLiveModel.findOne(query) + } + toFormattedJSON (canSeePrivateInformation: boolean): LiveVideo { let privateInformation: Pick | {} = {} diff --git a/server/core/models/video/video-playlist-element.ts b/server/core/models/video/video-playlist-element.ts index 62b0e3434..008f97a40 100644 --- a/server/core/models/video/video-playlist-element.ts +++ b/server/core/models/video/video-playlist-element.ts @@ -29,7 +29,8 @@ import { MVideoPlaylistElementAP, MVideoPlaylistElementFormattable, MVideoPlaylistElementVideoUrlPlaylistPrivacy, - MVideoPlaylistVideoThumbnail + MVideoPlaylistElementVideoThumbnail, + MVideoPlaylistElementVideoUrl } from '@server/types/models/video/video-playlist-element.js' import { AttributesOnly } from '@peertube/peertube-typescript-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' @@ -214,6 +215,26 @@ export class VideoPlaylistElementModel extends Model { + const query = { + order: getSort('position'), + where: { + videoPlaylistId + }, + include: [ + { + model: VideoModel.scope(VideoScopeNames.WITH_THUMBNAILS), + required: true + } + ] + } + + return VideoPlaylistElementModel + .findOne(query) + } + + // --------------------------------------------------------------------------- + static listUrlsOfForAP (videoPlaylistId: number, start: number, count: number, t?: Transaction) { const getQuery = (forCount: boolean) => { return { @@ -239,24 +260,25 @@ export class VideoPlaylistElementModel extends Model { + static listElementsForExport (videoPlaylistId: number): Promise { const query = { - order: getSort('position'), where: { videoPlaylistId }, include: [ { - model: VideoModel.scope(VideoScopeNames.WITH_THUMBNAILS), + attributes: [ 'url' ], + model: VideoModel.unscoped(), required: true } ] } - return VideoPlaylistElementModel - .findOne(query) + return VideoPlaylistElementModel.findAll(query) } + // --------------------------------------------------------------------------- + static getNextPositionOf (videoPlaylistId: number, transaction?: Transaction) { const query: AggregateOptions = { where: { diff --git a/server/core/models/video/video-playlist.ts b/server/core/models/video/video-playlist.ts index 716e78c4c..bea5a10e2 100644 --- a/server/core/models/video/video-playlist.ts +++ b/server/core/models/video/video-playlist.ts @@ -48,9 +48,9 @@ import { } from '../../initializers/constants.js' import { MThumbnail } from '../../types/models/video/thumbnail.js' import { + MVideoPlaylist, MVideoPlaylistAccountThumbnail, - MVideoPlaylistAP, - MVideoPlaylistFormattable, + MVideoPlaylistAP, MVideoPlaylistFormattable, MVideoPlaylistFull, MVideoPlaylistFullSummary, MVideoPlaylistSummaryWithElements @@ -497,6 +497,20 @@ export class VideoPlaylistModel extends Model { + const query = { + where: { + ownerAccountId: accountId + } + } + + return VideoPlaylistModel + .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) + .findAll(query) + } + + // --------------------------------------------------------------------------- + static doesPlaylistExist (url: string) { const query = { attributes: [ 'id' ], @@ -558,6 +572,32 @@ export class VideoPlaylistModel extends Model { + const query = { + where: { + type: VideoPlaylistType.WATCH_LATER, + ownerAccountId: account.id + } + } + + return VideoPlaylistModel + .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) + .findOne(query) + } + + static loadRegularByAccountAndName (account: MAccountId, name: string): Promise { + const query = { + where: { + type: VideoPlaylistType.REGULAR, + name, + ownerAccountId: account.id + } + } + + return VideoPlaylistModel + .findOne(query) + } + static getPrivacyLabel (privacy: VideoPlaylistPrivacyType) { return VIDEO_PLAYLIST_PRIVACIES[privacy] || 'Unknown' } diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts index ef6437513..c06708052 100644 --- a/server/core/models/video/video-streaming-playlist.ts +++ b/server/core/models/video/video-streaming-playlist.ts @@ -1,7 +1,7 @@ import { - VideoStorage, + FileStorage, VideoStreamingPlaylistType, - type VideoStorageType, + type FileStorageType, type VideoStreamingPlaylistType_Type } from '@peertube/peertube-models' import { sha1 } from '@peertube/peertube-node-utils' @@ -103,9 +103,9 @@ export class VideoStreamingPlaylistModel extends Model VideoModel, { foreignKey: { @@ -222,7 +222,7 @@ export class VideoStreamingPlaylistModel extends Model>> { return queryBuilder.queryVideo({ url, transaction, type: 'thumbnails' }) } - static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Promise { + static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ url, transaction, type: 'account' }) + } + + static loadByUrlAndPopulateAccountAndFiles (url: string, transaction?: Transaction): Promise { const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) return queryBuilder.queryVideo({ url, transaction, type: 'account-blacklist-files' }) @@ -1494,6 +1500,15 @@ export class VideoModel extends Model>> { } } + static loadByNameAndChannel (channel: MChannelId, name: string): Promise { + return VideoModel.unscoped().findOne({ + where: { + name, + channelId: channel.id + } + }) + } + static incrementViews (id: number, views: number) { return VideoModel.increment('views', { by: views, @@ -1929,7 +1944,7 @@ export class VideoModel extends Model>> { const promises: Promise[] = [ remove(filePath) ] if (!isRedundancy) promises.push(videoFile.removeTorrent()) - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + if (videoFile.storage === FileStorage.OBJECT_STORAGE) { promises.push(removeWebVideoObjectStorage(videoFile)) } @@ -1969,7 +1984,7 @@ export class VideoModel extends Model>> { streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) ) - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { await removeHLSObjectStorage(streamingPlaylist.withVideo(this)) } } @@ -1983,7 +1998,7 @@ export class VideoModel extends Model>> { const resolutionFilename = getHlsResolutionPlaylistFilename(videoFile.filename) await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + if (videoFile.storage === FileStorage.OBJECT_STORAGE) { await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename) await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename) } @@ -1993,7 +2008,7 @@ export class VideoModel extends Model>> { const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename) await remove(filePath) - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + if (streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) { await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename) } } diff --git a/server/core/types/express.d.ts b/server/core/types/express.d.ts index 3ead23b40..9fb333a45 100644 --- a/server/core/types/express.d.ts +++ b/server/core/types/express.d.ts @@ -1,6 +1,6 @@ import { OutgoingHttpHeaders } from 'http' import { Writable } from 'stream' -import { HttpMethodType, PeerTubeProblemDocumentData, ServerErrorCode, VideoCreate } from '@peertube/peertube-models' +import { HttpMethodType, PeerTubeProblemDocumentData, VideoCreate } from '@peertube/peertube-models' import { RegisterServerAuthExternalOptions } from '@server/types/index.js' import { MAbuseMessage, @@ -13,6 +13,7 @@ import { MRegistration, MStreamingPlaylist, MUserAccountUrl, + MUserExport, MVideoChangeOwnershipFull, MVideoFile, MVideoFormattableDetails, @@ -87,7 +88,7 @@ declare module 'express' { export type CustomUploadXFile = UploadXFile & { metadata: T } export type EnhancedUploadXFile = CustomUploadXFile & { - duration: number + duration?: number // If video file path: string filename: string originalname: string @@ -143,6 +144,7 @@ declare module 'express' { uploadVideoFileResumable?: UploadNewVideoUploadXFile updateVideoFileResumable?: EnhancedUploadXFile + importUserFileResumable?: EnhancedUploadXFile videoImport?: MVideoImportDefault @@ -217,6 +219,8 @@ declare module 'express' { runner?: MRunner runnerRegistrationToken?: MRunnerRegistrationToken runnerJob?: MRunnerJobRunner + + userExport?: MUserExport } } } diff --git a/server/core/types/models/user/index.ts b/server/core/types/models/user/index.ts index 53db3944d..c444e2f3c 100644 --- a/server/core/types/models/user/index.ts +++ b/server/core/types/models/user/index.ts @@ -1,4 +1,6 @@ export * from './user.js' +export * from './user-export.js' +export * from './user-import.js' export * from './user-notification.js' export * from './user-notification-setting.js' export * from './user-registration.js' diff --git a/server/core/types/models/user/user-export.ts b/server/core/types/models/user/user-export.ts new file mode 100644 index 000000000..0379655ba --- /dev/null +++ b/server/core/types/models/user/user-export.ts @@ -0,0 +1,5 @@ +import { UserExportModel } from '@server/models/user/user-export.js' + +// ############################################################################ + +export type MUserExport = Omit diff --git a/server/core/types/models/user/user-import.ts b/server/core/types/models/user/user-import.ts new file mode 100644 index 000000000..d97278665 --- /dev/null +++ b/server/core/types/models/user/user-import.ts @@ -0,0 +1,5 @@ +import { UserImportModel } from '@server/models/user/user-import.js' + +// ############################################################################ + +export type MUserImport = Omit diff --git a/server/core/types/models/video/video-caption.ts b/server/core/types/models/video/video-caption.ts index 8e8393d92..c00226ba5 100644 --- a/server/core/types/models/video/video-caption.ts +++ b/server/core/types/models/video/video-caption.ts @@ -11,7 +11,8 @@ export type MVideoCaption = Omit // ############################################################################ export type MVideoCaptionLanguage = Pick -export type MVideoCaptionLanguageUrl = Pick +export type MVideoCaptionLanguageUrl = + Pick export type MVideoCaptionVideo = MVideoCaption & diff --git a/server/core/types/models/video/video-comment.ts b/server/core/types/models/video/video-comment.ts index a7247911c..79c54a1e1 100644 --- a/server/core/types/models/video/video-comment.ts +++ b/server/core/types/models/video/video-comment.ts @@ -12,6 +12,13 @@ export type MCommentTotalReplies = MComment & { totalReplies?: number } export type MCommentId = Pick export type MCommentUrl = Pick +// --------------------------------------------------------------------------- + +export type MCommentExport = + Pick & + Use<'Video', MVideoAccountLight> & + Use<'InReplyToVideoComment', MCommentUrl> + // ############################################################################ export type MCommentOwner = diff --git a/server/core/types/models/video/video-live.ts b/server/core/types/models/video/video-live.ts index 6a3ca8f49..84b34a8cc 100644 --- a/server/core/types/models/video/video-live.ts +++ b/server/core/types/models/video/video-live.ts @@ -17,6 +17,10 @@ export type MVideoLiveVideo = // ############################################################################ +export type MVideoLiveWithSetting = + MVideoLive & + Use<'ReplaySetting', MLiveReplaySetting> + export type MVideoLiveVideoWithSetting = MVideoLiveVideo & Use<'ReplaySetting', MLiveReplaySetting> diff --git a/server/core/types/models/video/video-playlist-element.ts b/server/core/types/models/video/video-playlist-element.ts index bf8ff6daf..ea1552ac2 100644 --- a/server/core/types/models/video/video-playlist-element.ts +++ b/server/core/types/models/video/video-playlist-element.ts @@ -17,10 +17,14 @@ export type MVideoPlaylistElementLight = Pick +export type MVideoPlaylistElementVideoUrl = + MVideoPlaylistElement & + Use<'Video', MVideoUrl> + export type MVideoPlaylistElementVideoUrlPlaylistPrivacy = MVideoPlaylistElement & Use<'Video', MVideoUrl> & diff --git a/server/core/types/models/video/video-rate.ts b/server/core/types/models/video/video-rate.ts index 873b289e6..dc6504fb3 100644 --- a/server/core/types/models/video/video-rate.ts +++ b/server/core/types/models/video/video-rate.ts @@ -1,7 +1,7 @@ import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js' import { PickWith } from '@peertube/peertube-typescript-utils' import { MAccountAudience, MAccountUrl } from '../account/account.js' -import { MVideo, MVideoFormattable } from './video.js' +import { MVideo, MVideoFormattable, MVideoUrl } from './video.js' type Use = PickWith @@ -9,6 +9,10 @@ type Use = PickWith +export type MAccountVideoRateVideoUrl = + MAccountVideoRate & + Use<'Video', MVideoUrl> + export type MAccountVideoRateAccountUrl = MAccountVideoRate & Use<'Account', MAccountUrl> diff --git a/server/scripts/create-generate-storyboard-job.ts b/server/scripts/create-generate-storyboard-job.ts index 5c964ea13..c33b7e331 100644 --- a/server/scripts/create-generate-storyboard-job.ts +++ b/server/scripts/create-generate-storyboard-job.ts @@ -4,7 +4,7 @@ import { initDatabaseModels } from '@server/initializers/database.js' import { JobQueue } from '@server/lib/job-queue/index.js' import { StoryboardModel } from '@server/models/video/storyboard.js' import { VideoModel } from '@server/models/video/video.js' -import { buildStoryboardJobIfNeeded } from '@server/lib/video.js' +import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' program .description('Generate videos storyboard') diff --git a/server/scripts/create-move-video-storage-job.ts b/server/scripts/create-move-video-storage-job.ts index 42faf5779..2d17f602d 100644 --- a/server/scripts/create-move-video-storage-job.ts +++ b/server/scripts/create-move-video-storage-job.ts @@ -5,7 +5,7 @@ import { initDatabaseModels } from '@server/initializers/database.js' import { JobQueue } from '@server/lib/job-queue/index.js' import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js' import { VideoModel } from '@server/models/video/video.js' -import { VideoState, VideoStorage } from '@peertube/peertube-models' +import { VideoState, FileStorage } from '@peertube/peertube-models' import { MStreamingPlaylist, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' program @@ -84,7 +84,7 @@ async function run () { video: videoFull, type: 'to object storage', canProcessVideo: (files, hls) => { - return files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM + return files.some(f => f.storage === FileStorage.FILE_SYSTEM) || hls?.storage === FileStorage.FILE_SYSTEM }, handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined }) }) @@ -98,7 +98,7 @@ async function run () { type: 'to file system', canProcessVideo: (files, hls) => { - return files.some(f => f.storage === VideoStorage.OBJECT_STORAGE) || hls?.storage === VideoStorage.OBJECT_STORAGE + return files.some(f => f.storage === FileStorage.OBJECT_STORAGE) || hls?.storage === FileStorage.OBJECT_STORAGE }, handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined }) }) diff --git a/server/server.ts b/server/server.ts index 8c65c9425..ed16faed8 100644 --- a/server/server.ts +++ b/server/server.ts @@ -147,6 +147,7 @@ import { isTestOrDevInstance } from '@peertube/peertube-node-utils' import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js' import { ApplicationModel } from '@server/models/application/application.js' import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler.js' +import { RemoveExpiredUserExportsScheduler } from '@server/lib/schedulers/remove-expired-user-exports-scheduler.js' // ----------- Command line ----------- @@ -326,6 +327,7 @@ async function startApplication () { VideoViewsBufferScheduler.Instance.enable() GeoIPUpdateScheduler.Instance.enable() RunnerJobWatchDogScheduler.Instance.enable() + RemoveExpiredUserExportsScheduler.Instance.enable() OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) diff --git a/tsconfig.base.json b/tsconfig.base.json index cce74235b..9ccf78d51 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -19,7 +19,6 @@ "es2018", "es2019" ], - "baseUrl": "./", "resolveJsonModule": true, "strict": false, "strictBindCallApply": true, diff --git a/yarn.lock b/yarn.lock index cf41bebff..7f8be3063 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2282,6 +2282,13 @@ resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276" integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A== +"@types/archiver@^6.0.2": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2" + integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw== + dependencies: + "@types/readdir-glob" "*" + "@types/bcrypt@^5.0.0": version "5.0.0" resolved "https://registry.yarnpkg.com/@types/bcrypt/-/bcrypt-5.0.0.tgz#a835afa2882d165aff5690893db314eaa98b9f20" @@ -2482,6 +2489,13 @@ resolved "https://registry.yarnpkg.com/@types/jsonld/-/jsonld-1.5.10.tgz#f01cd5d5bc62dba508438b2f8ae7c9f35f0d8cd1" integrity sha512-fEXxh7TnXYTPcbHb/p3bJ2V5CGpzwv7wCwQ4GsvyvbSufn8mVrCXNuw/L1WVFkBC9qUl4UgDA4LRV6B8SmAiEw== +"@types/jsonwebtoken@^9.0.5": + version "9.0.5" + resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-9.0.5.tgz#0bd9b841c9e6c5a937c17656e2368f65da025588" + integrity sha512-VRLSGzik+Unrup6BsouBeHsf4d1hOEgYWTm/7Nmw1sXoN1+tRly/Gy/po3yeahnP4jfnQWWAhQAqcNfH7ngOkA== + dependencies: + "@types/node" "*" + "@types/lodash-es@^4.17.8": version "4.17.9" resolved "https://registry.yarnpkg.com/@types/lodash-es/-/lodash-es-4.17.9.tgz#49dbe5112e23c54f2b387d860b7d03028ce170c2" @@ -2644,6 +2658,13 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.5.tgz#38bd1733ae299620771bd414837ade2e57757498" integrity sha512-xrO9OoVPqFuYyR/loIHjnbvvyRZREYKLjxV4+dY6v3FQR3stQ9ZxIGkaclF7YhI9hfjpuTbu14hZEy94qKLtOA== +"@types/readdir-glob@*": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@types/readdir-glob/-/readdir-glob-1.1.5.tgz#21a4a98898fc606cb568ad815f2a0eedc24d412a" + integrity sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg== + dependencies: + "@types/node" "*" + "@types/request@^2.0.3": version "2.48.10" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.10.tgz#53dca1bb357849bab6e7d9c0888408c99f79a97c" @@ -2747,6 +2768,13 @@ dependencies: "@types/node" "*" +"@types/yauzl@^2.10.3": + version "2.10.3" + resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" + integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q== + dependencies: + "@types/node" "*" + "@typescript-eslint/eslint-plugin@^6.7.5": version "6.7.5" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz#f4024b9f63593d0c2b5bd6e4ca027e6f30934d4f" @@ -3054,6 +3082,31 @@ append-field@^1.0.0: resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc" integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== +archiver-utils@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-4.0.1.tgz#66ad15256e69589a77f706c90c6dbcc1b2775d2a" + integrity sha512-Q4Q99idbvzmgCTEAAhi32BkOyq8iVI5EwdO0PmBDSGIzzjYNdcFn7Q7k3OzbLy4kLUPXfJtG6fO2RjftXbobBg== + dependencies: + glob "^8.0.0" + graceful-fs "^4.2.0" + lazystream "^1.0.0" + lodash "^4.17.15" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + +archiver@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/archiver/-/archiver-6.0.1.tgz#d56968d4c09df309435adb5a1bbfc370dae48133" + integrity sha512-CXGy4poOLBKptiZH//VlWdFuUC1RESbdZjGjILwBuZ73P7WkAUN0htfSfBq/7k6FRFlpu7bg4JOkj1vU9G6jcQ== + dependencies: + archiver-utils "^4.0.1" + async "^3.2.4" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.1.2" + tar-stream "^3.0.0" + zip-stream "^5.0.1" + are-we-there-yet@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz#372e0e7bd279d8e94c653aaa1f67200884bf3e1c" @@ -3223,6 +3276,11 @@ async@^2.6.4: dependencies: lodash "^4.17.14" +async@^3.2.4: + version "3.2.5" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" + integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -3262,7 +3320,7 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -b4a@^1.3.1: +b4a@^1.3.1, b4a@^1.6.4: version "1.6.4" resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.6.4.tgz#ef1c1422cae5ce6535ec191baeed7567443f36c9" integrity sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw== @@ -3510,6 +3568,16 @@ browser-stdout@1.3.1: resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== +buffer-crc32@^0.2.1, buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== + +buffer-equal-constant-time@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" + integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA== + buffer-equal@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/buffer-equal/-/buffer-equal-0.0.1.tgz#91bc74b11ea405bc916bc6aa908faafa5b4aac4b" @@ -3978,6 +4046,16 @@ component-emitter@^1.3.0: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== +compress-commons@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-5.0.1.tgz#e46723ebbab41b50309b27a0e0f6f3baed2d6590" + integrity sha512-MPh//1cERdLtqwO3pOFLeXtpuai0Y2WCd5AhtKxznqM7WtaMYaOEMSgn45d9D10sIHSfIKE603HlOp8OPGrvag== + dependencies: + crc-32 "^1.2.0" + crc32-stream "^5.0.0" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -4107,6 +4185,19 @@ cpus@^1.0.3: resolved "https://registry.yarnpkg.com/cpus/-/cpus-1.0.3.tgz#4ef6deea461968d6329d07dd01205685df2934a2" integrity sha512-PXHBvGLuL69u55IkLa5e5838fLhIMHxmkV4ge42a8alGyn7BtawYgI0hQ849EedvtHIOLNNH3i6eQU1BiE9SUA== +crc-32@^1.2.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + +crc32-stream@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-5.0.0.tgz#a97d3a802c8687f101c27cc17ca5253327354720" + integrity sha512-B0EPa1UK+qnpBZpG+7FgPCu0J2ETLpXq09o9BkLkEAhdB6Z61Qo4pJ3JYu0c+Qi+/SAL7QThqnzS06pmSSyZaw== + dependencies: + crc-32 "^1.2.0" + readable-stream "^3.4.0" + create-torrent@^6.0.15: version "6.0.15" resolved "https://registry.yarnpkg.com/create-torrent/-/create-torrent-6.0.15.tgz#386d206ed4f417c8808b3e6f6d90142190e7b584" @@ -4544,6 +4635,13 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +ecdsa-sig-formatter@1.0.11: + version "1.0.11" + resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" + integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== + dependencies: + safe-buffer "^5.0.1" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -5265,7 +5363,7 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-fifo@^1.1.0: +fast-fifo@^1.1.0, fast-fifo@^1.2.0: version "1.3.2" resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c" integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ== @@ -5329,6 +5427,13 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fd-slicer@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" + integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== + dependencies: + pend "~1.2.0" + fecha@^4.2.0: version "4.2.3" resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" @@ -5743,7 +5848,7 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^8.0.3: +glob@^8.0.0, glob@^8.0.3: version "8.1.0" resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== @@ -6140,6 +6245,11 @@ immediate-chunk-store@^2.2.0: dependencies: queue-microtask "^1.2.3" +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== + import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" @@ -6639,6 +6749,22 @@ jsonpointer.js@0.4.0: resolved "https://registry.yarnpkg.com/jsonpointer.js/-/jsonpointer.js-0.4.0.tgz#002cb123f767aafdeb0196132ce5c4f9941ccaba" integrity sha512-2bf/1crAmPpsmj1I6rDT6W0SOErkrNBpb555xNWcMVWYrX6VnXpG0GRMQ2shvOHwafpfse8q0gnzPFYVH6Tqdg== +jsonwebtoken@^9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3" + integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ== + dependencies: + jws "^3.2.2" + lodash.includes "^4.3.0" + lodash.isboolean "^3.0.3" + lodash.isinteger "^4.0.4" + lodash.isnumber "^3.0.3" + lodash.isplainobject "^4.0.6" + lodash.isstring "^4.0.1" + lodash.once "^4.0.0" + ms "^2.1.1" + semver "^7.5.4" + jsprim@^1.2.2: version "1.4.2" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" @@ -6662,6 +6788,16 @@ jstransformer@1.0.0: is-promise "^2.0.0" promise "^7.0.1" +jszip@^3.10.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" + integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + setimmediate "^1.0.5" + juice@^9.0.0: version "9.1.0" resolved "https://registry.yarnpkg.com/juice/-/juice-9.1.0.tgz#3ef8a12392d44c1cd996022aa977581049a65050" @@ -6678,6 +6814,23 @@ junk@^4.0.1: resolved "https://registry.yarnpkg.com/junk/-/junk-4.0.1.tgz#7ee31f876388c05177fe36529ee714b07b50fbed" integrity sha512-Qush0uP+G8ZScpGMZvHUiRfI0YBWuB3gVBYlI0v0vvOJt5FLicco+IkP0a50LqTTQhmts/m6tP5SWE+USyIvcQ== +jwa@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a" + integrity sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + +jws@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304" + integrity sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA== + dependencies: + jwa "^1.4.1" + safe-buffer "^5.0.1" + k-bucket@^5.0.0, k-bucket@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/k-bucket/-/k-bucket-5.1.0.tgz#db2c9e72bd168b432e3f3e8fc092e2ccb61bff89" @@ -6734,6 +6887,13 @@ last-one-wins@^1.0.4: resolved "https://registry.yarnpkg.com/last-one-wins/-/last-one-wins-1.0.4.tgz#c1bfd0cbcb46790ec9156b8d1aee8fcb86cda22a" integrity sha512-t+KLJFkHPQk8lfN6WBOiGkiUXoub+gnb2XTYI2P3aiISL+94xgZ1vgz1SXN/N4hthuOoLXarXfBZPUruyjQtfA== +lazystream@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.1.tgz#494c831062f1f9408251ec44db1cba29242a2638" + integrity sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw== + dependencies: + readable-stream "^2.0.5" + leac@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/leac/-/leac-0.6.0.tgz#dcf136e382e666bd2475f44a1096061b70dc0912" @@ -6782,6 +6942,13 @@ libqp@2.0.1: resolved "https://registry.yarnpkg.com/libqp/-/libqp-2.0.1.tgz#b8fed76cc1ea6c9ceff8888169e4e0de70cd5cf2" integrity sha512-Ka0eC5LkF3IPNQHJmYBWljJsw0UvM6j+QdKRbWyCdTmYwvIDE6a7bCm0UkTAL/K+3KXK5qXT/ClcInU01OpdLg== +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + limiter@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/limiter/-/limiter-1.1.5.tgz#8f92a25b3b16c6131293a0cc834b4a838a2aa7c2" @@ -6864,17 +7031,52 @@ lodash.flatten@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" integrity sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g== +lodash.includes@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" + integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w== + lodash.isarguments@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== +lodash.isboolean@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" + integrity sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg== + +lodash.isinteger@^4.0.4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz#619c0af3d03f8b04c31f5882840b77b11cd68343" + integrity sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA== + +lodash.isnumber@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc" + integrity sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw== + +lodash.isplainobject@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" + integrity sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA== + +lodash.isstring@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" + integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw== + lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash@^4.17.14, lodash@^4.17.20, lodash@^4.17.21: +lodash.once@^4.0.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" + integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== + +lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -7196,7 +7398,7 @@ minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -minimatch@^5.0.1: +minimatch@^5.0.1, minimatch@^5.1.0: version "5.1.6" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== @@ -7835,7 +8037,7 @@ packet-reader@1.0.0: resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== -pako@^1.0.11, pako@^1.0.3: +pako@^1.0.11, pako@^1.0.3, pako@~1.0.2: version "1.0.11" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== @@ -8011,6 +8213,11 @@ peek-stream@^1.1.1: duplexify "^3.5.0" through2 "^2.0.3" +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== + pg-cloudflare@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" @@ -8593,7 +8800,7 @@ read@1.0.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@~2.3.6: +readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.2.2, readable-stream@~2.3.6: version "2.3.8" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== @@ -8629,6 +8836,13 @@ readable-wrap@^1.0.0: dependencies: readable-stream "^1.1.13-1" +readdir-glob@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.3.tgz#c3d831f51f5e7bfa62fa2ffbe4b508c640f09584" + integrity sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA== + dependencies: + minimatch "^5.1.0" + readdirp@^3.5.0, readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -8842,7 +9056,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@^5.1.0, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -9336,6 +9550,14 @@ streamx@^2.10.3, streamx@^2.13.2, streamx@^2.15.1: fast-fifo "^1.1.0" queue-tick "^1.0.1" +streamx@^2.15.0: + version "2.15.6" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.15.6.tgz#28bf36997ebc7bf6c08f9eba958735231b833887" + integrity sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw== + dependencies: + fast-fifo "^1.1.0" + queue-tick "^1.0.1" + string-argv@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" @@ -9523,6 +9745,15 @@ swagger-cli@^4.0.2: dependencies: "@apidevtools/swagger-cli" "4.0.4" +tar-stream@^3.0.0: + version "3.1.6" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.6.tgz#6520607b55a06f4a2e2e04db360ba7d338cc5bab" + integrity sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg== + dependencies: + b4a "^1.6.4" + fast-fifo "^1.2.0" + streamx "^2.15.0" + tar@^6.1.11: version "6.2.0" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.0.tgz#b14ce49a79cb1cd23bc9b016302dea5474493f73" @@ -10466,7 +10697,24 @@ yargs@^17.7.2: y18n "^5.0.5" yargs-parser "^21.1.1" +yauzl@^2.10.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== + dependencies: + buffer-crc32 "~0.2.3" + fd-slicer "~1.1.0" + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== + +zip-stream@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-5.0.1.tgz#cf3293bba121cad98be2ec7f05991d81d9f18134" + integrity sha512-UfZ0oa0C8LI58wJ+moL46BDIMgCQbnsb+2PoiJYtonhBsMh2bq1eRBVkvjfVsqbEHd9/EgKPUuL9saSSsec8OA== + dependencies: + archiver-utils "^4.0.1" + compress-commons "^5.0.1" + readable-stream "^3.6.0"