Merge pull request #4980 from emqx/merge-4.3-to-5.0

Merge 4.3 to 5.0

.ci/build_packages/tests.sh

@@ -150,17 +150,20 @@ relup_test(){
     if [ -d "${RELUP_PACKAGE_PATH}" ];then
         cd "${RELUP_PACKAGE_PATH}"
 
-        for var in "${EMQX_NAME}"-*-"${ARCH}".zip;do
+        find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
-            packagename=$(basename "${var}")
+            while read -r pkg; do
-            unzip "$packagename"
+                packagename=$(basename "${pkg}")
-            ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
+                unzip "$packagename"
-            ./emqx/bin/emqx versions
+                ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
-            cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
+                ./emqx/bin/emqx_ctl status
-            ./emqx/bin/emqx install "${TARGET_VERSION}"
+                ./emqx/bin/emqx versions
-            [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
+                cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
-            ./emqx/bin/emqx stop
+                ./emqx/bin/emqx install "${TARGET_VERSION}"
-            rm -rf emqx
+                [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
-        done
+                ./emqx/bin/emqx_ctl status
+                ./emqx/bin/emqx stop
+                rm -rf emqx
+            done
    fi
 }
 

.ci/fvt_tests/.env

@@ -1 +0,0 @@
-TARGET=emqx/emqx

.ci/fvt_tests/http_server/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, 
  [
-    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.1"}}}
+    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}}
  ]}.
 
 {shell, [

.ci/fvt_tests/relup.lux

@@ -28,6 +28,7 @@
     !cd emqx
     !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
     !sed -i '/emqx_telemetry/d' data/loaded_plugins
+
     !./bin/emqx start
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
@@ -40,6 +41,7 @@
     !cd emqx2
 
     !sed -i '/emqx_telemetry/d' data/loaded_plugins
+
     !./bin/emqx start
     ?EMQ X (.*) is started successfully!
     ?SH-PROMPT
@@ -72,15 +74,22 @@
 
 [shell bench]
     !cd $BENCH_PATH
+
     !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx]
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !echo "" > log/emqx.log.1
-    !./bin/emqx install $VSN
     ?SH-PROMPT
-    !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]"
+
-    ?$VSN
+    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
     ?SH-PROMPT
 
     !./bin/emqx_ctl cluster status
@@ -91,11 +100,17 @@
     ?SH-PROMPT
 
 [shell emqx2]
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !echo "" > log/emqx.log.1
-    !./bin/emqx install $VSN
     ?SH-PROMPT
-    !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]"
+
-    ?$VSN
+    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
     ?SH-PROMPT
 
     !./bin/emqx_ctl cluster status

.github/workflows/build_packages.yaml

@@ -133,7 +133,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - erl23.2.7.2-emqx-2
+          - 23.2.7.2-emqx-2
         exclude:
           - profile: emqx-edge
 
@@ -158,7 +158,11 @@ jobs:
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
+      env:
+        KERL_BUILD_BACKEND: git
+        OTP_GITHUB_URL: https://github.com/emqx/otp
       run: |
+        kerl update releases
         kerl build ${{ matrix.erl_otp }}
         kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
     - name: build

.github/workflows/build_slim_packages.yaml

@@ -81,8 +81,11 @@ jobs:
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
+      env:
+        KERL_BUILD_BACKEND: git
+        OTP_GITHUB_URL: https://github.com/emqx/otp
       run: |
-        export OTP_GITHUB_URL="https://github.com/emqx/otp"
+        kerl update releases
         kerl build ${{ matrix.erl_otp }}
         kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
     - name: build

.github/workflows/run_fvt_tests.yaml

@@ -191,7 +191,7 @@ jobs:
         - uses: actions/checkout@v2
           with:
             repository: hawk/lux
-            ref: lux-2.4
+            ref: lux-2.6
             path: lux
         - uses: actions/checkout@v2
           with:
@@ -253,14 +253,15 @@ jobs:
             make
             make install
         - name: run relup test
+          timeout-minutes: 20
           run: |
             set -e -x -u
             if [ -n "$OLD_VSNS" ]; then
                 mkdir -p packages
                 cp emqx/_packages/${PROFILE}/*.zip packages
                 cp emqx/_upgrade_base/*.zip packages
-                lux -v \
+                lux \
-                --timeout 600000 \
+                --case_timeout infinity \
                 --var PROFILE=$PROFILE \
                 --var PACKAGE_PATH=$(pwd)/packages \
                 --var BENCH_PATH=$(pwd)/emqtt-bench \
@@ -275,5 +276,3 @@ jobs:
             name: lux_logs
             path: lux_logs
 
-
-

rebar.config

@@ -33,7 +33,7 @@
 
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
-    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.5"}}}
+    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.6"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}

scripts/apps-version-check.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-remote="refs/remote/$(git remote -v | grep fetch | grep 'emqx/emqx' | awk '{print $1}')"
+latest_release=$(git describe --abbrev=0 --tags)
-latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1 --remotes="$remote")")
 
 bad_app_count=0