diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..2f69eb1d2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,100 @@ +image: chocobozzz/peertube-ci:10 + +stages: + - build-and-lint + - test + - nightly + +before_script: + - 'sed -i -z "s/database:\n hostname: ''localhost''/database:\n hostname: ''postgres''/" config/test.yaml' + - 'sed -i -z "s/redis:\n hostname: ''localhost''/redis:\n hostname: ''redis''/" config/test.yaml' + - if [[ $CI_JOB_STAGE == "test" ]]; then psql -c "create user peertube with password 'peertube';"; fi + - NOCLIENT=1 yarn install --pure-lockfile --cache-folder .yarn-cache + +cache: + key: yarn + paths: + - .yarn-cache + - cached-fixtures + +### +## Jobs templates +# +#.build-and-lint: &build-and-lint +# stage: build-and-lint +# +#.tests: &tests +# stage: test +# dependencies: +# - build-server +# services: +# - name: postgres:9.6 +# alias: postgres +# - name: redis:latest +# alias: redis +# variables: +# PGHOST: postgres +# PGUSER: postgres +# REDIS_HOST: redis +# artifacts: +# expire_in: 1 day +# paths: +# - test*/logs +# when: always +# +#### +### Build and lint +## +#build-server: +# <<: *build-and-lint +# artifacts: +# expire_in: 5h +# paths: +# - dist/ +# script: +# - npm run build:server +# +#lint: +# <<: *build-and-lint +# script: +# - yarn install --pure-lockfile --cache-folder .yarn-cache +# - npm run ci -- "lint" +# +#### +### Tests +# +#test-misc: +# <<: *tests +# script: +# - yarn install --pure-lockfile --cache-folder .yarn-cache +# - npm run ci -- "misc" +# +#test-cli: +# <<: *tests +# retry: +# max: 1 +# script: +# - npm run ci -- "cli" +# +#api: +# <<: *tests +# parallel: 4 +# retry: +# max: 1 +# script: +# - NODE_PENDING_JOB_WAIT=1000 npm run ci -- api-$CI_NODE_INDEX + +build-nightly: + stage: nightly + only: + - schedules + script: + - yarn install --pure-lockfile --cache-folder .yarn-cache + - npm run nightly + - mkdir "${HOME}/.ssh" + - chmod 700 "${HOME}/.ssh" + - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi + - eval `ssh-agent -s` + - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi + - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi + diff --git a/.travis.yml b/.travis.yml index 9018b082a..3d287a782 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: - env: TEST_SUITE=lint script: - - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run travis -- "$TEST_SUITE" + - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE" after_failure: - cat test1/logs/peertube.log diff --git a/README.md b/README.md index 3edaeef2f..5de29a067 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
- - Build Status + + pipeline status diff --git a/config/test.yaml b/config/test.yaml index e5ac74f57..8843bb2dc 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -20,6 +20,9 @@ database: hostname: 'localhost' port: 5432 +redis: + hostname: 'localhost' + smtp: hostname: null port: 1025 diff --git a/package.json b/package.json index 300f54d9f..e8821bc70 100644 --- a/package.json +++ b/package.json @@ -60,17 +60,19 @@ "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", "tsc": "tsc", "commander": "commander", - "lint": "npm run travis -- lint", + "lint": "npm run ci -- lint", "ng": "ng", "nodemon": "nodemon", "ts-node": "ts-node", "tslint": "tslint", "concurrently": "concurrently", + "mocha-parallel-tests": "mocha-parallel-tests", "sasslint": "sass-lint --verbose --no-exit", "sasslint:fix": "sass-lint-auto-fix -c .sass-lint.yml --verbose", "mocha": "mocha", - "travis": "scripty", + "ci": "scripty", "release": "scripty", + "nightly": "scripty", "client-report": "scripty" }, "husky": { @@ -93,7 +95,7 @@ ] }, "resolutions": { - "@types/bluebird": "3.5.21" + "@types/bluebird": "3.5.27" }, "dependencies": { "apicache": "^1.4.0", @@ -206,7 +208,7 @@ "maildev": "^1.0.0-rc3", "marked-man": "^0.6.0", "mocha": "^6.0.0", - "mocha-parallel-tests": "^2.1.0", + "mocha-parallel-tests": "^2.2.1", "nodemon": "^1.18.6", "sass-lint": "^1.12.1", "source-map-support": "^0.5.0", diff --git a/scripts/travis.sh b/scripts/ci.sh similarity index 86% rename from scripts/travis.sh rename to scripts/ci.sh index 42e2329c6..1819f73dd 100755 --- a/scripts/travis.sh +++ b/scripts/ci.sh @@ -22,16 +22,16 @@ elif [ "$1" = "cli" ]; then mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts elif [ "$1" = "api-1" ]; then npm run build:server - sh ./server/tests/api/travis-1.sh 2 + sh ./server/tests/api/ci-1.sh 2 elif [ "$1" = "api-2" ]; then npm run build:server - sh ./server/tests/api/travis-2.sh 2 + sh ./server/tests/api/ci-2.sh 2 elif [ "$1" = "api-3" ]; then npm run build:server - sh ./server/tests/api/travis-3.sh 2 + sh ./server/tests/api/ci-3.sh 2 elif [ "$1" = "api-4" ]; then npm run build:server - sh ./server/tests/api/travis-4.sh 2 + sh ./server/tests/api/ci-4.sh 2 elif [ "$1" = "lint" ]; then npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts" diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 20745170d..dbd399aaa 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -5,7 +5,7 @@ set -eu recreateDB () { dbname="peertube_test$1" - dropdb --if-exists "$dbname" + dropdb --if-exists "$dbname" 2>&1 createdb -O peertube "$dbname" psql -c "CREATE EXTENSION pg_trgm;" "$dbname" & @@ -18,10 +18,15 @@ removeFiles () { dropRedis () { port=$((9000+$1)) + host="localhost" - redis-cli KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL - redis-cli KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL - redis-cli KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli DEL + if [ ! -z ${GITLAB_CI+x} ]; then + host="redis" + fi + + redis-cli -h "$host" KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL + redis-cli -h "$host" KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL + redis-cli -h "$host" KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL } seq=$(seq 1 6) diff --git a/scripts/nightly.sh b/scripts/nightly.sh new file mode 100755 index 000000000..bde5d57b9 --- /dev/null +++ b/scripts/nightly.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -eu + +shutdown() { + # Get our process group id + # shellcheck disable=SC2009 + PGID=$(ps -o pgid= $$ | grep -o "[0-9]*") + + # Kill it in a new new process group + setsid kill -- -"$PGID" + exit 0 +} + +trap "shutdown" SIGINT SIGTERM + +today=$(date '+%F') +directory_name="peertube-nightly-$today" +tar_name="peertube-nightly-$today.tar.xz" + +npm run build + +# Creating the archives +( + # local variables + directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \ + "$directory_name/LICENSE" "$directory_name/README.md" \ + "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \ + "$directory_name/client/package.json" "$directory_name/config" \ + "$directory_name/dist" "$directory_name/package.json" \ + "$directory_name/scripts" "$directory_name/support" \ + "$directory_name/tsconfig.json" "$directory_name/yarn.lock") + + # temporary setup + cd .. + ln -s "PeerTube" "$directory_name" + + XZ_OPT=-e9 tar cfJ "PeerTube/$tar_name" "${directories_to_archive[@]}" + + # temporary setup destruction + rm "$directory_name" +) diff --git a/scripts/test.sh b/scripts/test.sh index 5ec7a5920..1b6b29b66 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,6 +5,6 @@ set -eu npm run build:server npm run setup:cli -npm run travis -- lint +npm run ci -- lint mocha --exit --require ts-node/register/type-check --bail server/tests/index.ts diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index ca70230a2..77808466c 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts @@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { // We send the video abuse to the origin server if (videoInstance.isOwned() === false) { - await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance) + await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) } - Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) - auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) return videoAbuseInstance }) + Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse) + logger.info('Abuse report for video %s created.', videoInstance.name) return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end() diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index 39c74b2fd..6c5068fb0 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -2,6 +2,7 @@ import * as retry from 'async/retry' import * as Bluebird from 'bluebird' import { Model } from 'sequelize-typescript' import { logger } from './logger' +import { Transaction } from 'sequelize' function retryTransactionWrapper ( functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise | Bluebird, @@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model, savedFields: object) { }) } +function afterCommitIfTransaction (t: Transaction, fn: Function) { + if (t) return t.afterCommit(() => fn()) + + return fn() +} + // --------------------------------------------------------------------------- export { resetSequelizeInstance, retryTransactionWrapper, transactionRetryer, - updateInstanceWithAnother + updateInstanceWithAnother, + afterCommitIfTransaction } diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts index 0b3976089..8faab051e 100644 --- a/server/lib/activitypub/process/process-flag.ts +++ b/server/lib/activitypub/process/process-flag.ts @@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object }) - return sequelizeTypescript.transaction(async t => { + const videoAbuse = await sequelizeTypescript.transaction(async t => { const videoAbuseData = { reporterAccountId: account.id, reason: flag.content, @@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) videoAbuseInstance.Video = video - Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) - logger.info('Remote abuse for video uuid %s created', flag.object) + + return videoAbuseInstance }) + + Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse) } diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 28f18595b..9c21149f2 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc import { VideoPlaylistModel } from '../../../models/video/video-playlist' import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' import { getServerActor } from '../../../helpers/utils' +import * as Bluebird from 'bluebird' async function sendCreateVideo (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException) + await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException) // Broadcast to our followers await broadcastToFollowers(createActivity, byActor, [ byActor ], t) @@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException) // Send to origin - return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) + t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)) } function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 7bf5ca520..7a1d6f0ba 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts @@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) + await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException) // Broadcast to our followers await broadcastToFollowers(activity, byActor, [ byActor ], t) @@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException) // Send to origin - return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) + t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)) } async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) { diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts index 96a7311b9..61ee389a6 100644 --- a/server/lib/activitypub/send/send-flag.ts +++ b/server/lib/activitypub/send/send-flag.ts @@ -6,8 +6,9 @@ import { unicastTo } from './utils' import { logger } from '../../../helpers/logger' import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' import { audiencify, getAudience } from '../audience' +import { Transaction } from 'sequelize' -async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { +async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) { if (!video.VideoChannel.Account.Actor.serverId) return // Local user const url = getVideoAbuseActivityPubUrl(videoAbuse) @@ -18,7 +19,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) - return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)) } function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index 2c3d02014..c6e7fe83d 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts @@ -4,8 +4,9 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { getActorFollowActivityPubUrl } from '../url' import { unicastTo } from './utils' import { logger } from '../../../helpers/logger' +import { Transaction } from 'sequelize' -function sendFollow (actorFollow: ActorFollowModel) { +function sendFollow (actorFollow: ActorFollowModel, t: Transaction) { const me = actorFollow.ActorFollower const following = actorFollow.ActorFollowing @@ -17,7 +18,7 @@ function sendFollow (actorFollow: ActorFollowModel) { const url = getActorFollowActivityPubUrl(me, following) const data = buildFollowActivity(url, me, following) - return unicastTo(data, me, following.inboxUrl) + t.afterCommit(() => unicastTo(data, me, following.inboxUrl)) } function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 8727a121e..8fcbbac5c 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -37,7 +37,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const followActivity = buildFollowActivity(followUrl, me, following) const undoActivity = undoActivityData(undoUrl, me, followActivity) - return unicastTo(undoActivity, me, following.inboxUrl) + t.afterCommit(() => unicastTo(undoActivity, me, following.inboxUrl)) } async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 7411c08d5..5bf092894 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -26,7 +26,9 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) // Needed to build the AP object - if (!video.VideoCaptions) video.VideoCaptions = await video.$get('VideoCaptions') as VideoCaptionModel[] + if (!video.VideoCaptions) { + video.VideoCaptions = await video.$get('VideoCaptions', { transaction: t }) as VideoCaptionModel[] + } const videoObject = video.toActivityPubObject() const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index 69706e620..1faae1d84 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts @@ -7,6 +7,7 @@ import { JobQueue } from '../../job-queue' import { VideoModel } from '../../../models/video/video' import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' import { getServerActor } from '../../../helpers/utils' +import { afterCommitIfTransaction } from '../../../helpers/database-utils' async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { byActor: ActorModel, @@ -20,7 +21,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo) const activity = activityBuilder(audience) - return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl) + return afterCommitIfTransaction(options.transaction, () => { + return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl) + }) } // Send to followers @@ -28,6 +31,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud const activity = activityBuilder(audience) const actorsException = [ options.byActor ] + return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException) } @@ -76,7 +80,7 @@ async function forwardActivity ( uris, body: activity } - return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }) + return afterCommitIfTransaction(t, () => JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })) } async function broadcastToFollowers ( @@ -87,20 +91,22 @@ async function broadcastToFollowers ( actorsException: ActorModel[] = [] ) { const uris = await computeFollowerUris(toFollowersOf, actorsException, t) - return broadcastTo(uris, data, byActor) + + return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor)) } async function broadcastToActors ( data: any, byActor: ActorModel, toActors: ActorModel[], + t?: Transaction, actorsException: ActorModel[] = [] ) { const uris = await computeUris(toActors, actorsException) - return broadcastTo(uris, data, byActor) + return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor)) } -async function broadcastTo (uris: string[], data: any, byActor: ActorModel) { +function broadcastTo (uris: string[], data: any, byActor: ActorModel) { if (uris.length === 0) return undefined logger.debug('Creating broadcast job.', { uris }) @@ -114,7 +120,7 @@ async function broadcastTo (uris: string[], data: any, byActor: ActorModel) { return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }) } -async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) { +function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) { logger.debug('Creating unicast job.', { uri: toActorUrl }) const payload = { @@ -123,7 +129,7 @@ async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) { body: data } - return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload }) + JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload }) } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index b3defb617..4ae66cd01 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts @@ -69,7 +69,7 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) { actorFollow.ActorFollower = fromActor // Send a notification to remote server if our follow is not already accepted - if (actorFollow.state !== 'accepted') await sendFollow(actorFollow) + if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t) return actorFollow }) diff --git a/server/tests/api/activitypub/fetch.ts b/server/tests/api/activitypub/fetch.ts index 3a1c0d321..3d54c2042 100644 --- a/server/tests/api/activitypub/fetch.ts +++ b/server/tests/api/activitypub/fetch.ts @@ -87,7 +87,7 @@ describe('Test ActivityPub fetcher', function () { }) after(async function () { - this.timeout(10000) + this.timeout(20000) await cleanupTests(servers) diff --git a/server/tests/api/travis-1.sh b/server/tests/api/ci-1.sh similarity index 79% rename from server/tests/api/travis-1.sh rename to server/tests/api/ci-1.sh index db4021b25..8dd9756a5 100644 --- a/server/tests/api/travis-1.sh +++ b/server/tests/api/ci-1.sh @@ -6,5 +6,5 @@ checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo) searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo) -MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \ +MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \ $notificationsFiles $searchFiles $checkParamFiles diff --git a/server/tests/api/travis-2.sh b/server/tests/api/ci-2.sh similarity index 63% rename from server/tests/api/travis-2.sh rename to server/tests/api/ci-2.sh index ba7a061b0..16ab585e9 100644 --- a/server/tests/api/travis-2.sh +++ b/server/tests/api/ci-2.sh @@ -5,5 +5,5 @@ set -eu serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo) usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo) -MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \ +MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \ $serverFiles $usersFiles diff --git a/server/tests/api/travis-3.sh b/server/tests/api/ci-3.sh similarity index 58% rename from server/tests/api/travis-3.sh rename to server/tests/api/ci-3.sh index 82457222c..fc96f6fb4 100644 --- a/server/tests/api/travis-3.sh +++ b/server/tests/api/ci-3.sh @@ -4,5 +4,5 @@ set -eu videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo) -MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \ +MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \ $videosFiles diff --git a/server/tests/api/travis-4.sh b/server/tests/api/ci-4.sh similarity index 66% rename from server/tests/api/travis-4.sh rename to server/tests/api/ci-4.sh index 875986182..74809e1ad 100644 --- a/server/tests/api/travis-4.sh +++ b/server/tests/api/ci-4.sh @@ -5,5 +5,5 @@ set -eu redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo) activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo) -MOCHA_PARALLEL=true mocha-parallel-tests --max-parallel $1 --timeout 5000 --exit --require ts-node/register --bail \ +MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \ $redundancyFiles $activitypubFiles diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts index 662b64e05..6fa630562 100644 --- a/server/tests/api/notifications/user-notifications.ts +++ b/server/tests/api/notifications/user-notifications.ts @@ -782,7 +782,7 @@ describe('Test users notifications', function () { it('Should not send a notification before the video is published', async function () { this.timeout(20000) - let updateAt = new Date(new Date().getTime() + 100000) + let updateAt = new Date(new Date().getTime() + 1000000) const data = { privacy: VideoPrivacy.PRIVATE, @@ -1074,7 +1074,7 @@ describe('Test users notifications', function () { it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { this.timeout(20000) - let updateAt = new Date(new Date().getTime() + 100000) + let updateAt = new Date(new Date().getTime() + 1000000) const name = 'video with auto-blacklist and future schedule ' + uuidv4() diff --git a/server/tests/api/server/email.ts b/server/tests/api/server/email.ts index 7b7acfd12..c55a221f2 100644 --- a/server/tests/api/server/email.ts +++ b/server/tests/api/server/email.ts @@ -40,7 +40,7 @@ describe('Test emails', function () { let emailPort: number before(async function () { - this.timeout(30000) + this.timeout(50000) emailPort = await MockSmtpServer.Instance.collectEmails(emails) diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index 068654d8c..a225443c5 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -109,7 +109,7 @@ describe('Test handle downs', function () { }) it('Should remove followers that are often down', async function () { - this.timeout(60000) + this.timeout(240000) // Server 2 and 3 follow server 1 await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken) diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index 3ab2fe120..ceea47a85 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts @@ -8,6 +8,7 @@ import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../.. import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers' import { uploadVideo } from '../../../../shared/extra-utils/videos/videos' import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs' +import { Job } from '../../../../shared/models/server' const expect = chai.expect @@ -50,7 +51,7 @@ describe('Test jobs', function () { if (job.type === 'videos-views') job = res.body.data[1] expect(job.state).to.equal('completed') - expect(job.type).to.equal('activitypub-follow') + expect(job.type.startsWith('activitypub-')).to.be.true expect(dateIsValid(job.createdAt)).to.be.true expect(dateIsValid(job.processedOn)).to.be.true expect(dateIsValid(job.finishedOn)).to.be.true diff --git a/server/tests/api/videos/video-schedule-update.ts b/server/tests/api/videos/video-schedule-update.ts index 64f657780..65a8eafb8 100644 --- a/server/tests/api/videos/video-schedule-update.ts +++ b/server/tests/api/videos/video-schedule-update.ts @@ -85,7 +85,7 @@ describe('Test video update scheduler', function () { }) it('Should wait some seconds and have the video in public privacy', async function () { - this.timeout(20000) + this.timeout(50000) await wait(15000) await waitJobs(servers) diff --git a/shared/extra-utils/miscs/miscs.ts b/shared/extra-utils/miscs/miscs.ts index 42250886c..08f41a7e7 100644 --- a/shared/extra-utils/miscs/miscs.ts +++ b/shared/extra-utils/miscs/miscs.ts @@ -1,10 +1,10 @@ /* tslint:disable:no-unused-expression */ import * as chai from 'chai' -import { basename, isAbsolute, join, resolve } from 'path' +import { basename, dirname, isAbsolute, join, resolve } from 'path' import * as request from 'supertest' import * as WebTorrent from 'webtorrent' -import { pathExists, readFile } from 'fs-extra' +import { ensureDir, pathExists, readFile } from 'fs-extra' import * as ffmpeg from 'fluent-ffmpeg' const expect = chai.expect @@ -59,12 +59,12 @@ async function testImage (url: string, imageName: string, imagePath: string, ext expect(data.length).to.be.below(maxLength) } -function buildAbsoluteFixturePath (path: string, customTravisPath = false) { +function buildAbsoluteFixturePath (path: string, customCIPath = false) { if (isAbsolute(path)) { return path } - if (customTravisPath && process.env.TRAVIS) return join(process.env.HOME, 'fixtures', path) + if (customCIPath && process.env.GITLAB_CI) return join(root(), 'cached-fixtures', path) return join(root(), 'server', 'tests', 'fixtures', path) } @@ -72,6 +72,8 @@ function buildAbsoluteFixturePath (path: string, customTravisPath = false) { async function generateHighBitrateVideo () { const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) + await ensureDir(dirname(tempFixturePath)) + const exists = await pathExists(tempFixturePath) if (!exists) { diff --git a/shared/extra-utils/miscs/sql.ts b/shared/extra-utils/miscs/sql.ts index 1961a8762..dfe840d8c 100644 --- a/shared/extra-utils/miscs/sql.ts +++ b/shared/extra-utils/miscs/sql.ts @@ -1,6 +1,5 @@ import { QueryTypes, Sequelize } from 'sequelize' import { ServerInfo } from '../server/servers' -import { PluginType } from '../../models/plugins/plugin.type' let sequelizes: { [ id: number ]: Sequelize } = {} @@ -10,7 +9,7 @@ function getSequelize (internalServerNumber: number) { const dbname = 'peertube_test' + internalServerNumber const username = 'peertube' const password = 'peertube' - const host = 'localhost' + const host = process.env.GITLAB_CI ? 'postgres' : 'localhost' const port = 5432 const seq = new Sequelize(dbname, username, password, { diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 692b5e24d..11b570f60 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts @@ -2,6 +2,7 @@ import * as request from 'supertest' import { Job, JobState } from '../../models' import { wait } from '../miscs/miscs' import { ServerInfo } from './servers' +import { inspect } from 'util' function getJobsList (url: string, accessToken: string, state: JobState) { const path = '/api/v1/jobs/' + state @@ -49,7 +50,9 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { .then(res => res.body.data) .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) .then(jobs => { - if (jobs.length !== 0) pendingRequests = true + if (jobs.length !== 0) { + pendingRequests = true + } }) tasks.push(p) } diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 40cf7f0f3..a0720d778 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -79,8 +79,8 @@ function flushTests (serverNumber?: number) { return new Promise((res, rej) => { const suffix = serverNumber ? ` -- ${serverNumber}` : '' - return exec('npm run clean:server:test' + suffix, err => { - if (err) return rej(err) + return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => { + if (err || stderr) return rej(err || new Error(stderr)) return res() }) diff --git a/support/doc/development/release.md b/support/doc/development/release.md index 352a1f0d3..c03075735 100644 --- a/support/doc/development/release.md +++ b/support/doc/development/release.md @@ -10,7 +10,7 @@ * `NODE_APP_INSTANCE=6 NODE_ENV=test npm run start` and check migrations does not fail * Run `rm -r node_modules && rm -r client/node_modules && yarn install --pure-lockfile && npm run build` to see if all the supported languages compile correctly * Update https://peertube2.cpy.re and check it works correctly - * Check Travis tests are green + * Check CI tests are green * Run E2E tests: `BROWSERSTACK_USER=my_user BROWSERSTACK_KEY=my_key npm run e2e` * Release: `GITHUB_TOKEN=my_token npm run release -- 1.x.x` * Create a dedicated branch: `git checkout -b release/1.x.x && git push origin release/1.x.x` diff --git a/yarn.lock b/yarn.lock index 3d6b1466a..894d5e81b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -54,7 +54,12 @@ dependencies: "@types/node" "*" -"@types/bluebird@*", "@types/bluebird@3.5.21": +"@types/bluebird@*", "@types/bluebird@3.5.27": + version "3.5.27" + resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.27.tgz#61eb4d75dc6bfbce51cf49ee9bbebe941b2cb5d0" + integrity sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ== + +"@types/bluebird@3.5.21": version "3.5.21" resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.21.tgz#567615589cc913e84a28ecf9edb031732bdf2634" integrity sha512-6UNEwyw+6SGMC/WMI0ld0PS4st7Qq51qgguFrFizOSpGvZiqe9iswztFSdZvwJBEhLOy2JaxNE6VC7yMAlbfyQ== @@ -4678,7 +4683,7 @@ mkdirp@0.5.1, mkdirp@0.x.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: dependencies: minimist "0.0.8" -mocha-parallel-tests@^2.1.0: +mocha-parallel-tests@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/mocha-parallel-tests/-/mocha-parallel-tests-2.2.1.tgz#616b316c9f20c956a6ce18cd38d45e7db3045b99" integrity sha512-9t0NtQ2V/KrNKaUj08RVZp0dqJxeRMTGvC7JA5Daw/LA8GOP+4P7NxRgU0e2sM0JRWYxSakXtpE+MEVGX1ifuw==