diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh
new file mode 100755
index 000000000..7343c448b
--- /dev/null
+++ b/.ci/fvt_tests/local_relup_test_run.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+USAGE="$0 profile vsn old_vsn package_path"
+EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages"
+
+if [[ $# -ne 4 ]]; then
+    echo "$USAGE"
+    echo "$EXAMPLE"
+    exit 1
+fi
+
+set -ex
+
+PROFILE="$1"
+VSN="$2"
+OLD_VSN="$3"
+PACKAGE_PATH="$4"
+
+TEMPDIR=$(mktemp -d)
+trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
+
+git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx"
+cp -r "$PACKAGE_PATH" "$TEMPDIR/packages"
+cp relup.lux "$TEMPDIR/"
+cp -r http_server "$TEMPDIR/http_server"
+
+exec docker run \
+    -v "$TEMPDIR:/relup_test" \
+    -w "/relup_test" \
+    -e REBAR_COLOR=none \
+    -it savonarola/emqx-relup-env:4.3 \
+        lux \
+        --case_timeout infinity \
+        --var PROFILE="$PROFILE" \
+        --var PACKAGE_PATH="/relup_test/packages" \
+        --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
+        --var VSN="$VSN" \
+        --var OLD_VSN="$OLD_VSN" \
+        relup.lux
+
diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index aa179c6cf..574b0e885 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -1,15 +1,12 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
-[config var=BENCH_PATH]
 [config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
-[config var=OLD_VSNS]
+[config var=OLD_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
 
-[loop old_vsn $OLD_VSNS]
-
 [shell http_server]
     !cd http_server
     !rebar3 shell
@@ -22,7 +19,7 @@
 
 [shell emqx]
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed  -r 's/[v|e]//g')-amd64.zip
+    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-amd64.zip
     ?SH-PROMPT
 
     !cd emqx
@@ -81,9 +78,7 @@
     ???SH-PROMPT
 
 [shell bench]
-    !cd $BENCH_PATH
-
-    !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
+    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx]
@@ -107,7 +102,7 @@
     """
     ?SH-PROMPT
 
-    !./bin/emqx_ctl plugins list | grep emqx_management
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
     ?Plugin\(emqx_management.*active=true\)
     ?SH-PROMPT
 
@@ -132,7 +127,7 @@
     """
     ?SH-PROMPT
 
-    !./bin/emqx_ctl plugins list | grep emqx_management
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
     ?Plugin\(emqx_management.*active=true\)
     ?SH-PROMPT
 
@@ -190,8 +185,6 @@
     !halt(3).
     ?SH-PROMPT:
 
-[endloop]
-
 [cleanup]
     !echo ==$$?==
     ?==0==
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index db8a64e93..2c97844f6 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -178,76 +178,78 @@ jobs:
             fi
             exit $RESULT
 
-    relup_test:
+    relup_test_plan:
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        outputs:
+          profile: ${{ steps.profile-and-versions.outputs.profile }}
+          vsn: ${{ steps.profile-and-versions.outputs.vsn }}
+          old_vsns: ${{ steps.profile-and-versions.outputs.old_vsns }}
+          broker: ${{ steps.profile-and-versions.outputs.broker }}
+          matrix: ${{ steps.generate-matrix.outputs.matrix }}
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/checkout@v2
+          name: Checkout
+          with:
+            path: emqx
+            fetch-depth: 0
+        - name: Get profile and version list
+          id: profile-and-versions
+          run: |
+            cd emqx
+            vsn="$(./pkg-vsn.sh)"
+            pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
+
+            if make emqx-ee --dry-run > /dev/null 2>&1; then
+              profile="emqx-ee"
+              old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
+              broker="emqx-ee"
+
+            else
+              profile="emqx"
+              old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
+              broker="emqx-ce"
+            fi
+
+            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
+
+            echo "::set-output name=vsn::$vsn"
+            echo "::set-output name=profile::$profile"
+            echo "::set-output name=broker::$broker"
+            echo "::set-output name=old_vsns::$old_vsns"
+        - name: Generate matrix
+          id: generate-matrix
+          run: |
+            matrix=$(echo -n "$OLD_VSNS" | jq -R -s -c 'split(" ")')
+            echo "::set-output name=matrix::$matrix"
+
+    relup_test_build:
+        needs: relup_test_plan
         runs-on: ubuntu-20.04
         container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
         defaults:
           run:
             shell: bash
+        env:
+          OLD_VSNS: "${{ needs.relup_test_plan.outputs.old_vsns }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
         steps:
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.8'
-            architecture: 'x64'
         - uses: actions/checkout@v2
+          name: Checkout
           with:
-            repository: emqx/paho.mqtt.testing
-            ref: develop-4.0
-            path: paho.mqtt.testing
-        - uses: actions/checkout@v2
-          with:
-            repository: terry-xiaoyu/one_more_emqx
-            ref: master
-            path: one_more_emqx
-        - uses: actions/checkout@v2
-          with:
-            repository: emqx/emqtt-bench
-            ref: '0.3.3'
-            path: emqtt-bench
-        - uses: actions/checkout@v2
-          with:
-            repository: hawk/lux
-            ref: lux-2.6
-            path: lux
-        - uses: actions/checkout@v2
-          with:
-            repository: ${{ github.repository }}
             path: emqx
-            fetch-depth: 0
-        - name: prepare
+        - name: Prepare credentials
           run: |
-            if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
+            if [ "$PROFILE" = "emqx-ee" ]; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
-              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
-            else
-              echo "PROFILE=emqx" >> $GITHUB_ENV
             fi
-        - name: get version
-          run: |
-            set -e -x -u
-            cd emqx
-            if [ $PROFILE = "emqx" ];then
-                broker="emqx-ce"
-                edition='opensource'
-            else
-                broker="emqx-ee"
-                edition='enterprise'
-            fi
-            echo "BROKER=$broker" >> $GITHUB_ENV
-
-            vsn="$(./pkg-vsn.sh)"
-            echo "VSN=$vsn" >> $GITHUB_ENV
-
-            pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-            if [ $PROFILE = "emqx" ]; then
-                old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            else
-                old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            fi
-            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-        - name: download emqx
+        - name: Download bases
           run: |
             set -e -x -u
             mkdir -p emqx/_upgrade_base
@@ -256,53 +258,70 @@ jobs:
             for old_vsn in ${old_vsns[@]}; do
               wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
             done
-        - name: build emqx
+        - name: Build emqx
           run: make -C emqx ${PROFILE}-zip
         - uses: actions/upload-artifact@v2
+          name: Upload built emqx and test scenario
           with:
-            name: emqx
-            path: emqx/_packages/**/*.zip
-        - name: build emqtt-bench
-          run: make -C emqtt-bench
-        - name: build lux
-          run: |
-            set -e -u -x
-            cd lux
-            autoconf
-            ./configure
-            make
-            make install
-        - name: run relup test
-          timeout-minutes: 30
+            name: emqx_built
+            path: |
+              emqx/_packages/*/*.zip
+              emqx/.ci/fvt_tests
+
+    relup_test_run:
+        needs:
+          - relup_test_plan
+          - relup_test_build
+        runs-on: ubuntu-20.04
+        container: emqx/relup-test-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        strategy:
+          matrix:
+            old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
+        env:
+          OLD_VSN: "${{ matrix.old_vsn }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          VSN: "${{ needs.relup_test_plan.outputs.vsn }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/download-artifact@v2
+          name: Dowload built emqx and test scenario
+          with:
+            name: emqx_built
+            path: emqx_built
+        - uses: actions/checkout@v2
+          name: Checkout one_more_emqx
+          with:
+            repository: terry-xiaoyu/one_more_emqx
+            ref: master
+            path: one_more_emqx
+        - name: Prepare packages
           run: |
             set -e -x -u
-            if [ -n "$OLD_VSNS" ]; then
-                mkdir -p packages
-                cp emqx/_packages/${PROFILE}/*.zip packages
-                cp emqx/_upgrade_base/*.zip packages
-                lux \
-                --case_timeout infinity \
-                --var PROFILE=$PROFILE \
-                --var PACKAGE_PATH=$(pwd)/packages \
-                --var BENCH_PATH=$(pwd)/emqtt-bench \
-                --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
-                --var VSN="$VSN" \
-                --var OLD_VSNS="$OLD_VSNS" \
-                emqx/.ci/fvt_tests/relup.lux
-            fi
-        - uses: actions/upload-artifact@v1
+            mkdir -p packages
+            cp emqx_built/_packages/*/*.zip packages
+            cd packages
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip
+        - name: Run relup test scenario
+          timeout-minutes: 5
+          run: |
+            lux \
+            --case_timeout infinity \
+            --var PROFILE=$PROFILE \
+            --var PACKAGE_PATH=$(pwd)/packages \
+            --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
+            --var VSN="$VSN" \
+            --var OLD_VSN="$OLD_VSN" \
+            emqx_built/.ci/fvt_tests/relup.lux
+        - uses: actions/upload-artifact@v2
+          name: Save debug data
           if: failure()
           with:
-            name: emqx_logs
-            path: packages/emqx/log/emqx.log.1
-        - uses: actions/upload-artifact@v1
-          if: failure()
-          with:
-            name: emqx2_logs
-            path: packages/emqx2/log/emqx.log.1
-        - uses: actions/upload-artifact@v1
-          if: failure()
-          with:
-            name: lux_logs
-            path: lux_logs
-
+            name: debug_data
+            path: |
+              packages/emqx/log/emqx.log.1
+              packages/emqx2/log/emqx.log.1
+              packages/*.zip
+              lux_logs