interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix sync import of latest videos

This commit is contained in:
Chocobozzz 2022-09-16 09:37:07 +02:00
parent c95fbe6553
commit 3204f4d17d
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
5 changed files with 106 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/helpers/youtube-dl/youtube-dl-info-builder.ts
View File

@ -12,7 +12,7 @@ type YoutubeDLInfo = {
tags?: string[] tags?: string[]
thumbnailUrl?: string thumbnailUrl?: string
ext?: string ext?: string
originallyPublishedAt?: Date originallyPublishedAtWithoutTime?: Date
webpageUrl?: string webpageUrl?: string
urls?: string[] urls?: string[]
@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder {
tags: this.getTags(obj.tags), tags: this.getTags(obj.tags),
thumbnailUrl: obj.thumbnail || undefined, thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj), urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj), originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
ext: obj.ext, ext: obj.ext,
webpageUrl: obj.webpage_url webpageUrl: obj.webpage_url
} }

4
server/lib/job-queue/handlers/video-channel-import.ts
View File

@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
import { VideoChannelModel } from '@server/models/video/video-channel' import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { MChannelSync } from '@server/types/models' import { MChannelSync } from '@server/types/models'
import { VideoChannelImportPayload } from '@shared/models' import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'
export async function processVideoChannelImport (job: Job) { export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload const payload = job.data as VideoChannelImportPayload
@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) {
}) })
} catch (err) { } catch (err) {
logger.error(`Failed to import channel ${videoChannel.name}`, { err }) logger.error(`Failed to import channel ${videoChannel.name}`, { err })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
} }
} }

5
server/lib/sync-channel.ts
View File

@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat
const videoInfo = await youtubeDL.getInfoForDownload() const videoInfo = await youtubeDL.getInfoForDownload()
if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) { const onlyAfterWithoutTime = new Date(onlyAfter)
onlyAfterWithoutTime.setHours(0, 0, 0)
if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) {
return true return true
} }
} }

2
server/lib/video-import.ts
View File

@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
channelId, channelId,
originallyPublishedAt: importDataOverride?.originallyPublishedAt originallyPublishedAt: importDataOverride?.originallyPublishedAt
? new Date(importDataOverride?.originallyPublishedAt) ? new Date(importDataOverride?.originallyPublishedAt)
: importData.originallyPublishedAt : importData.originallyPublishedAtWithoutTime
} }
videoData = await Hooks.wrapObject( videoData = await Hooks.wrapObject(

138
server/tests/api/videos/video-channel-syncs.ts
View File

@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils' import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
import { import {
ChannelSyncsCommand, createMultipleServers,
createSingleServer,
getServerImportConfig, getServerImportConfig,
killallServers,
PeerTubeServer, PeerTubeServer,
setAccessTokensToServers, setAccessTokensToServers,
setDefaultAccountAvatar, setDefaultAccountAvatar,
@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () {
function runSuite (mode: 'youtube-dl' | 'yt-dlp') { function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
describe('Sync using ' + mode, function () { describe('Sync using ' + mode, function () {
let server: PeerTubeServer let servers: PeerTubeServer[]
let command: ChannelSyncsCommand
let startTestDate: Date let startTestDate: Date
@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () {
} }
async function changeDateForSync (channelSyncId: number, newDate: string) { async function changeDateForSync (channelSyncId: number, newDate: string) {
await server.sql.updateQuery( await servers[0].sql.updateQuery(
`UPDATE "videoChannelSync" ` + `UPDATE "videoChannelSync" ` +
`SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
`WHERE id=${channelSyncId}` `WHERE id=${channelSyncId}`
@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () {
} }
before(async function () { before(async function () {
this.timeout(120_000) this.timeout(240_000)
startTestDate = new Date() startTestDate = new Date()
server = await createSingleServer(1, getServerImportConfig(mode)) servers = await createMultipleServers(2, getServerImportConfig(mode))
await setAccessTokensToServers([ server ]) await setAccessTokensToServers(servers)
await setDefaultVideoChannel([ server ]) await setDefaultVideoChannel(servers)
await setDefaultChannelAvatar([ server ]) await setDefaultChannelAvatar(servers)
await setDefaultAccountAvatar([ server ]) await setDefaultAccountAvatar(servers)
await server.config.enableChannelSync() await servers[0].config.enableChannelSync()
command = server.channelSyncs
{ {
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken }) const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
userInfo.channelId = videoChannels[0].id userInfo.channelId = videoChannels[0].id
} }
}) })
@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () {
this.timeout(120_000) this.timeout(120_000)
{ {
const { video } = await server.imports.importVideo({ const { video } = await servers[0].imports.importVideo({
attributes: { attributes: {
channelId: server.store.channel.id, channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC, privacy: VideoPrivacy.PUBLIC,
targetUrl: FIXTURE_URLS.youtube targetUrl: FIXTURE_URLS.youtube
} }
@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () {
expect(video.name).to.equal('small video - youtube') expect(video.name).to.equal('small video - youtube')
const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
expect(total).to.equal(1) expect(total).to.equal(1)
} }
const { videoChannelSync } = await command.create({ const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: { attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel, externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: server.store.channel.id videoChannelId: servers[0].store.channel.id
}, },
token: server.accessToken, token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200 expectedStatus: HttpStatusCode.OK_200
}) })
rootChannelSyncId = videoChannelSync.id rootChannelSyncId = videoChannelSync.id
@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () {
// Ensure any missing video not already fetched will be considered as new // Ensure any missing video not already fetched will be considered as new
await changeDateForSync(videoChannelSync.id, '1970-01-01') await changeDateForSync(videoChannelSync.id, '1970-01-01')
await server.debug.sendCommand({ await servers[0].debug.sendCommand({
body: { body: {
command: 'process-video-channel-sync-latest' command: 'process-video-channel-sync-latest'
} }
}) })
{ {
await waitJobs(server) await waitJobs(servers)
const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) const { total, data } = await servers[0].videos.listByChannel({
handle: 'root_channel',
include: VideoInclude.NOT_PUBLISHED_STATE
})
expect(total).to.equal(2) expect(total).to.equal(2)
expect(data[0].name).to.equal('test') expect(data[0].name).to.equal('test')
} }
@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () {
it('Should add another synchronization', async function () { it('Should add another synchronization', async function () {
const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
const { videoChannelSync } = await command.create({ const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: { attributes: {
externalChannelUrl, externalChannelUrl,
videoChannelId: server.store.channel.id videoChannelId: servers[0].store.channel.id
}, },
token: server.accessToken, token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200 expectedStatus: HttpStatusCode.OK_200
}) })
expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
expect(videoChannelSync.channel).to.include({ expect(videoChannelSync.channel).to.include({
id: server.store.channel.id, id: servers[0].store.channel.id,
name: 'root_channel' name: 'root_channel'
}) })
expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () {
}) })
it('Should add a synchronization for another user', async function () { it('Should add a synchronization for another user', async function () {
const { videoChannelSync } = await command.create({ const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: { attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
videoChannelId: userInfo.channelId videoChannelId: userInfo.channelId
@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () {
}) })
it('Should not import a channel if not asked', async function () { it('Should not import a channel if not asked', async function () {
await waitJobs(server) await waitJobs(servers)
const { data } = await command.listByAccount({ accountName: userInfo.username }) const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(data[0].state).to.contain({ expect(data[0].state).to.contain({
id: VideoChannelSyncState.WAITING_FIRST_RUN, id: VideoChannelSyncState.WAITING_FIRST_RUN,
@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () {
await changeDateForSync(userInfo.syncId, '2019-03-01') await changeDateForSync(userInfo.syncId, '2019-03-01')
await server.debug.sendCommand({ await servers[0].debug.sendCommand({
body: { body: {
command: 'process-video-channel-sync-latest' command: 'process-video-channel-sync-latest'
} }
}) })
await waitJobs(server) await waitJobs(servers)
const { data, total } = await server.videos.listByChannel({ const { data, total } = await servers[0].videos.listByChannel({
handle: userInfo.channelName, handle: userInfo.channelName,
include: VideoInclude.NOT_PUBLISHED_STATE include: VideoInclude.NOT_PUBLISHED_STATE
}) })
@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () {
it('Should list channel synchronizations', async function () { it('Should list channel synchronizations', async function () {
// Root // Root
{ {
const { total, data } = await command.listByAccount({ accountName: 'root' }) const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
expect(total).to.equal(2) expect(total).to.equal(2)
expect(data[0]).to.deep.contain({ expect(data[0]).to.deep.contain({
@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () {
expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
expect(data[0].channel).to.contain({ id: server.store.channel.id }) expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
} }
// User // User
{ {
const { total, data } = await command.listByAccount({ accountName: userInfo.username }) const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(1) expect(total).to.equal(1)
expect(data[0]).to.deep.contain({ expect(data[0]).to.deep.contain({
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () {
}) })
it('Should list imports of a channel synchronization', async function () { it('Should list imports of a channel synchronization', async function () {
const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
expect(total).to.equal(1) expect(total).to.equal(1)
expect(data).to.have.lengthOf(1) expect(data).to.have.lengthOf(1)
@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () {
}) })
it('Should remove user\'s channel synchronizations', async function () { it('Should remove user\'s channel synchronizations', async function () {
await command.delete({ channelSyncId: userInfo.syncId }) await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
const { total } = await command.listByAccount({ accountName: userInfo.username }) const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(0) expect(total).to.equal(0)
}) })
// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
// it('Should import a remote PeerTube channel', async function () {
// this.timeout(240_000)
// await servers[1].videos.quickUpload({ name: 'remote 1' })
// await waitJobs(servers)
// const { videoChannelSync } = await servers[0].channelSyncs.create({
// attributes: {
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelId: userInfo.channelId
// },
// token: userInfo.accessToken
// })
// await servers[0].channels.importVideos({
// channelName: userInfo.channelName,
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelSyncId: videoChannelSync.id,
// token: userInfo.accessToken
// })
// await waitJobs(servers)
// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })
// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 1')
// })
// it('Should keep synced a remote PeerTube channel', async function () {
// this.timeout(240_000)
// await servers[1].videos.quickUpload({ name: 'remote 2' })
// await waitJobs(servers)
// await servers[0].debug.sendCommand({
// body: {
// command: 'process-video-channel-sync-latest'
// }
// })
// await waitJobs(servers)
// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })
// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 2')
// })
after(async function () { after(async function () {
await server?.kill() await killallServers(servers)
}) })
}) })
} }