From 5d81f159d6834229ad00dedb433e9ddcefcc6e87 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Jun 2022 14:00:27 +0800
Subject: [PATCH 01/23] fix(CI): syntax errors in run_relup_tests.yaml

---
 .ci/fvt_tests/relup.lux                | 2 +-
 .github/workflows/run_relup_tests.yaml | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index f5b47b89e..9e67dfd63 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -22,7 +22,7 @@
 
 [shell emqx]
     !cd $PACKAGE_PATH
-    mkdir -p emqx
+    !mkdir -p emqx
     !tar -C emqx -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
     ?SH-PROMPT
 
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 97c33f229..9783d0781 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -70,8 +70,9 @@ jobs:
         fetch-depth: 0
 
     - name: Get old vsn
-      run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
-      run: echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
+      run: |
+        echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
+        echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
 
     - name: build emqx
       env:

From efffc450d0cf148b7b3d891da30676945fff6923 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Jun 2022 14:00:27 +0800
Subject: [PATCH 02/23] fix: update relup.lux

---
 .ci/fvt_tests/http_server/src/http_server.erl | 55 ++++++++++---------
 .ci/fvt_tests/relup.lux                       | 47 +++++++---------
 2 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl
index b66b72939..4b1981de4 100644
--- a/.ci/fvt_tests/http_server/src/http_server.erl
+++ b/.ci/fvt_tests/http_server/src/http_server.erl
@@ -1,36 +1,41 @@
 -module(http_server).
 
--import(minirest, [ return/0
-                  , return/1
-                  ]).
+-import(minirest, [
+    return/0,
+    return/1
+]).
 
--export([ start/0
-        , stop/0
-        ]).
+-export([
+    start/0,
+    stop/0
+]).
 
--rest_api(#{ name => get_counter 
-           , method => 'GET'
-           , path => "/counter"
-           , func => get_counter
-           , descr => "Check counter"
-           }).
--rest_api(#{ name => add_counter
-           , method => 'POST'
-           , path => "/counter"
-           , func => add_counter
-           , descr => "Counter plus one"
-           }).
+-rest_api(#{
+    name => get_counter,
+    method => 'GET',
+    path => "/counter",
+    func => get_counter,
+    descr => "Check counter"
+}).
+-rest_api(#{
+    name => add_counter,
+    method => 'POST',
+    path => "/counter",
+    func => add_counter,
+    descr => "Counter plus one"
+}).
 
--export([ get_counter/2
-        , add_counter/2
-        ]).
+-export([
+    get_counter/2,
+    add_counter/2
+]).
 
 start() ->
     application:ensure_all_started(minirest),
     ets:new(relup_test_message, [named_table, public]),
     Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}],
     Dispatch = [{"/[...]", minirest, Handlers}],
-    minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 8080}]}, Dispatch).
+    minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 7077}]}, Dispatch).
 
 stop() ->
     ets:delete(relup_test_message),
@@ -39,12 +44,12 @@ stop() ->
 get_counter(_Binding, _Params) ->
     return({ok, ets:info(relup_test_message, size)}).
 
-add_counter(_Binding, Params)  ->
+add_counter(_Binding, Params) ->
     case lists:keymember(<<"payload">>, 1, Params) of
-         true ->
+        true ->
             {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params),
             ets:insert(relup_test_message, {ID, Params1});
-         _ ->
+        _ ->
             ok
     end,
     return().
diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index 9e67dfd63..a6de27e94 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -16,7 +16,7 @@
     ???Eshell
     ???>
     !http_server:start().
-    ?Start http_server listener on 8080 successfully.
+    ?Start http_server listener on 7077 successfully.
     ?ok
     ?>
 
@@ -27,9 +27,6 @@
     ?SH-PROMPT
 
     !cd emqx
-    !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
-
     !./bin/emqx start
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
@@ -53,24 +50,27 @@
 
     !./bin/emqx_ctl cluster status
     """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
+    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []}
     """
     ?SH-PROMPT
 
-    !./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}'
-    ?created
+    ## create a webhook data bridge with id "my_webhook"
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","method":"post","url":"http://127.0.0.1:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status'
+    ?connected
     ?SH-PROMPT
-    !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
-    ?created
+
+    ## create a rule that uses the webhook as action, the rule id = "rule_edsy"
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n  *\nFROM\n  \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id'
+    ?rule_edsy
     ?SH-PROMPT
 
 [shell emqx]
-    !./bin/emqx_ctl resources list
-    ?691c29ba
+    ## verify the bridges and rules are sync to the other node
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges/webhook:my_webhook' -X 'GET' -H 'Content-Type: application/json' | jq '.name'
+    ?my_webhook
     ?SH-PROMPT
-    !./bin/emqx_ctl rules list
-    ?691c29ba
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules/rule_edsy' -X 'GET' -H 'Content-Type: application/json' | jq '.id'
+    ?rule_edsy
     ?SH-PROMPT
 
 [shell bench]
@@ -116,15 +116,10 @@
 
     !./bin/emqx_ctl cluster status
     """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
+    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []}
     """
     ?SH-PROMPT
 
-    !./bin/emqx_ctl plugins list | grep emqx_management
-    ?Plugin\(emqx_management.*active=true\)
-    ?SH-PROMPT
-
 [shell emqx2]
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
@@ -162,15 +157,10 @@
 
     !./bin/emqx_ctl cluster status
     """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
+    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []}
     """
     ?SH-PROMPT
 
-    !./bin/emqx_ctl plugins list | grep emqx_management
-    ?Plugin\(emqx_management.*active=true\)
-    ?SH-PROMPT
-
 ## We don't guarantee not to lose a single message!
 ## So even if we received 290~300 messages, we consider it as success
 [shell bench]
@@ -187,6 +177,11 @@
     ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
+    ## The /counter API is provided by .ci/fvt_test/http_server
+    !curl http://127.0.0.1:7077/counter
+    ?\{"data":(29[0-9])|(300),"code":0\}
+    ?SH-PROMPT
+
 [shell emqx2]
     !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
     -error

From 0d980904e09e0dd21a890cd7ae6ad3e7250ce563 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Jun 2022 22:16:03 +0800
Subject: [PATCH 03/23] fix: tmp - test relup

---
 .ci/fvt_tests/relup.lux | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index a6de27e94..f3ac60521 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -27,7 +27,7 @@
     ?SH-PROMPT
 
     !cd emqx
-    !./bin/emqx start
+    !./bin/emqx console
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
@@ -40,7 +40,7 @@
 
     !sed -i '/emqx_telemetry/d' data/loaded_plugins
 
-    !./bin/emqx start
+    !./bin/emqx console
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 

From 9ec88198cbe54e4d0a0959335b3b291ea64d1e81 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 23 Jun 2022 09:14:57 +0800
Subject: [PATCH 04/23] fix: remove one_more_emqx.sh from the relup.lux

---
 .ci/fvt_tests/relup.lux                | 17 +++++++++++------
 .github/workflows/run_relup_tests.yaml | 10 +++-------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index f3ac60521..1972d33ee 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -1,7 +1,6 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
 [config var=BENCH_PATH]
-[config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
 [config var=OLD_VSNS]
 
@@ -33,14 +32,20 @@
 
 [shell emqx2]
     !cd $PACKAGE_PATH
-    !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
-    !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
+    !cp -r emqx emqx2
     ?SH-PROMPT
     !cd emqx2
 
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
-
-    !./bin/emqx console
+    """!
+    EMQX_NODE__NAME='emqx2@127.0.0.1' \
+    EMQX_STATSD__SERVER='127.0.0.1:8124' \
+    EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1882' \
+    EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8882' \
+    EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8082' \
+    EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8085' \
+    EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18082' \
+    ./bin/emqx console
+    """
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 9783d0781..628684089 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -15,6 +15,8 @@ on:
 
 jobs:
   relup_test:
+    runs-on: aws-amd64
+
     strategy:
       matrix:
         profile:
@@ -32,13 +34,13 @@ jobs:
         arch:
           - amd64
 
-    runs-on: ubuntu-20.04
     container: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     defaults:
       run:
         shell: bash
     steps:
+    - uses: AutoModality/action-clean@v1
     - uses: actions/setup-python@v2
       with:
         python-version: '3.8'
@@ -48,11 +50,6 @@ jobs:
         repository: emqx/paho.mqtt.testing
         ref: develop-4.0
         path: paho.mqtt.testing
-    - uses: actions/checkout@v2
-      with:
-        repository: terry-xiaoyu/one_more_emqx
-        ref: master
-        path: one_more_emqx
     - uses: actions/checkout@v2
       with:
         repository: emqx/emqtt-bench
@@ -103,7 +100,6 @@ jobs:
             --var PROFILE=$PROFILE \
             --var PACKAGE_PATH=$(pwd)/packages \
             --var BENCH_PATH=$(pwd)/emqtt-bench \
-            --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
             --var OLD_VSNS="$OLD_VSNS" \
             emqx/.ci/fvt_tests/relup.lux

From 18f3920a73e6b2e87d69878d61dae39b301a1ee8 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 23 Jun 2022 21:59:11 +0200
Subject: [PATCH 05/23] feat: add --json option to 'emqx ctl cluster status'
 command

---
 apps/emqx_management/src/emqx_mgmt_cli.erl | 23 +++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl
index 50f4b21af..0f60317fa 100644
--- a/apps/emqx_management/src/emqx_mgmt_cli.erl
+++ b/apps/emqx_management/src/emqx_mgmt_cli.erl
@@ -135,14 +135,35 @@ cluster(["force-leave", SNode]) ->
     end;
 cluster(["status"]) ->
     emqx_ctl:print("Cluster status: ~p~n", [ekka_cluster:info()]);
+cluster(["status", "--json"]) ->
+    Info = sort_map_list_fields(ekka_cluster:info()),
+    emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Info)]);
 cluster(_) ->
     emqx_ctl:usage([
         {"cluster join <Node>", "Join the cluster"},
         {"cluster leave", "Leave the cluster"},
         {"cluster force-leave <Node>", "Force the node leave from cluster"},
-        {"cluster status", "Cluster status"}
+        {"cluster status [--json]", "Cluster status"}
     ]).
 
+%% sort lists for deterministic output
+sort_map_list_fields(Map) when is_map(Map) ->
+    lists:foldl(
+        fun(Field, Acc) ->
+            sort_map_list_field(Field, Acc)
+        end,
+        Map,
+        maps:keys(Map)
+    );
+sort_map_list_fields(NotMap) ->
+    NotMap.
+
+sort_map_list_field(Field, Map) ->
+    case maps:get(Field, Map) of
+        [_ | _] = L -> Map#{Field := lists:sort(L)};
+        _ -> Map
+    end.
+
 %%--------------------------------------------------------------------
 %% @doc Query clients
 

From bcefec19351d35efbb41ca564b80df586716e278 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 23 Jun 2022 21:59:43 +0200
Subject: [PATCH 06/23] ci: fix relup test

---
 .ci/fvt_tests/relup.lux                | 105 ++++++++++---------------
 .github/workflows/run_relup_tests.yaml |  19 +++--
 2 files changed, 49 insertions(+), 75 deletions(-)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index f5b47b89e..739e95180 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -20,58 +20,46 @@
     ?ok
     ?>
 
-[shell emqx]
+[shell emqx1]
     !cd $PACKAGE_PATH
-    mkdir -p emqx
-    !tar -C emqx -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !mkdir -p emqx1
+    !tar -C emqx1 -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
     ?SH-PROMPT
-
-    !cd emqx
-    !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
-
+    !cd emqx1
+    !export EMQX_NODE_NAME='emqx1@127.0.0.1'
     !./bin/emqx start
-    ?EMQ X .* is started successfully!
+    ?EMQX .* is started successfully!
     ?SH-PROMPT
 
 [shell emqx2]
     !cd $PACKAGE_PATH
-    !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
-    !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
+    !tar -C emqx2 -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !mkdir -p emqx2
     ?SH-PROMPT
     !cd emqx2
-
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
-
+    !export EMQX_NODE_NAME='emqx2@127.0.0.1'
+    !export EMQX_listeners__tcp__default__bind=2883
+    !export EMQX_listeners__ssl__default__bind=9883
+    !export EMQX_listeners__ws__default__bind=9083
+    !export EMQX_listeners__wss__default__bind=9084
     !./bin/emqx start
-    ?EMQ X .* is started successfully!
+    ?EMQX .* is started successfully!
     ?SH-PROMPT
 
-    !./bin/emqx_ctl cluster join emqx@127.0.0.1
+    !./bin/emqx_ctl cluster join emqx1@127.0.0.1
     ???Join the cluster successfully.
     ?SH-PROMPT
 
-    !./bin/emqx_ctl cluster status
-    """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
-    """
+    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
+    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
     ?SH-PROMPT
 
-    !./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}'
-    ?created
-    ?SH-PROMPT
-    !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
-    ?created
-    ?SH-PROMPT
-
-[shell emqx]
-    !./bin/emqx_ctl resources list
-    ?691c29ba
-    ?SH-PROMPT
-    !./bin/emqx_ctl rules list
-    ?691c29ba
-    ?SH-PROMPT
+# TODO find another way to create resource
+#    !./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}'
+#    ?created
+#    ?SH-PROMPT
+#    !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
+#    ?created
 
 [shell bench]
     !cd $BENCH_PATH
@@ -79,7 +67,7 @@
     !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
-[shell emqx]
+[shell emqx1]
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
@@ -90,7 +78,7 @@
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
 
-    !./bin/emqx versions |grep permanent
+    !./bin/emqx versions | grep permanent
     ?(.*)$VSN
     ?SH-PROMPT
 
@@ -99,7 +87,7 @@
     ?Made release permanent:.*
     ?SH-PROMPT
 
-    !./bin/emqx versions |grep permanent | grep -qs "$old_vsn"
+    !./bin/emqx versions | grep permanent | grep -qs "$old_vsn"
     ?SH-PROMPT:
     !echo ==$$?==
     ?^==0==
@@ -110,19 +98,12 @@
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
 
-    !./bin/emqx versions |grep permanent
+    !./bin/emqx versions | grep permanent
     ?(.*)$VSN
     ?SH-PROMPT
 
-    !./bin/emqx_ctl cluster status
-    """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
-    """
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl plugins list | grep emqx_management
-    ?Plugin\(emqx_management.*active=true\)
+    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
+    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
     ?SH-PROMPT
 
 [shell emqx2]
@@ -145,7 +126,7 @@
     ?Made release permanent:.*
     ?SH-PROMPT
 
-    !./bin/emqx versions |grep permanent | grep -qs "$old_vsn"
+    !./bin/emqx versions | grep permanent | grep -qs "$old_vsn"
     ?SH-PROMPT:
     !echo ==$$?==
     ?^==0==
@@ -160,15 +141,8 @@
     ?(.*)$VSN
     ?SH-PROMPT
 
-    !./bin/emqx_ctl cluster status
-    """???
-    Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
-                      stopped_nodes => []}
-    """
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl plugins list | grep emqx_management
-    ?Plugin\(emqx_management.*active=true\)
+    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
+    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
     ?SH-PROMPT
 
 ## We don't guarantee not to lose a single message!
@@ -179,13 +153,14 @@
     !sleep 5
     ?SH-PROMPT
 
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.matched"
-    ?300
-    ?SH-PROMPT
+# TODO adjust rule metrics query API
+#    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.matched"
+#    ?300
+#    ?SH-PROMPT
 
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.\"actions.success\""
-    ?\{"data":(29[0-9])|(300),"code":0\}
-    ?SH-PROMPT
+#    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.\"actions.success\""
+#    ?\{"data":(29[0-9])|(300),"code":0\}
+#    ?SH-PROMPT
 
 [shell emqx2]
     !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
@@ -199,7 +174,7 @@
     !rm -rf $PACKAGE_PATH/emqx2
     ?SH-PROMPT:
 
-[shell emqx]
+[shell emqx1]
     !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
     -error
     ??SH-PROMPT:
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 97c33f229..286c94c22 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -48,11 +48,6 @@ jobs:
         repository: emqx/paho.mqtt.testing
         ref: develop-4.0
         path: paho.mqtt.testing
-    - uses: actions/checkout@v2
-      with:
-        repository: terry-xiaoyu/one_more_emqx
-        ref: master
-        path: one_more_emqx
     - uses: actions/checkout@v2
       with:
         repository: emqx/emqtt-bench
@@ -70,8 +65,12 @@ jobs:
         fetch-depth: 0
 
     - name: Get old vsn
-      run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
-      run: echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
+      run: |
+        set -x
+        OLD_VSNS="$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)"
+        NOW_VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})
+        echo "OLD_VSNS=$OLD_VSNS" >> $GITHUB_ENV
+        echo "NOW_VSN=$NOW_VSN" >> $GITHUB_ENV
 
     - name: build emqx
       env:
@@ -95,15 +94,15 @@ jobs:
         set -e -x -u
         if [ -n "$OLD_VSNS" ]; then
             mkdir -p packages
-            cp emqx/_packages/${PROFILE}/*.tar.gz packages
-            cp emqx/_upgrade_base/*.tar.gz packages
+            cp emqx/_packages/${PROFILE}/*.tar.gz packages/
+            cp emqx/_upgrade_base/*.tar.gz packages/
             lux \
             --case_timeout infinity \
             --var PROFILE=$PROFILE \
             --var PACKAGE_PATH=$(pwd)/packages \
             --var BENCH_PATH=$(pwd)/emqtt-bench \
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
-            --var VSN="$VSN" \
+            --var VSN="$NOW_VSN" \
             --var OLD_VSNS="$OLD_VSNS" \
             emqx/.ci/fvt_tests/relup.lux
         fi

From 6aeb73821ff6da3d9aadcdea209403aed3c419ff Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 23 Jun 2022 23:11:51 +0200
Subject: [PATCH 07/23] ci: relup test upload packages after falure

---
 .github/workflows/run_relup_tests.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 286c94c22..205e761d4 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -111,3 +111,8 @@ jobs:
       with:
         name: lux_logs
         path: lux_logs
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: packages
+        path: packages

From d3f8ebd10f61094efb7fbc57cd02f45a00125a51 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 24 Jun 2022 06:09:48 +0200
Subject: [PATCH 08/23] ci: fix relup test workflow and upgrade builder to
 5.0-17

builder 5.0-17 has emqtt-bench and lux included
so we do not need to rebuild the software for each ci job
---
 .ci/docker-compose-file/docker-compose.yaml   |   2 +-
 .ci/fvt_tests/relup.lux                       |  32 ++-
 .../build_and_push_docker_images.yaml         |   4 +-
 .github/workflows/build_packages.yaml         |   4 +-
 .github/workflows/build_slim_packages.yaml    |   2 +-
 .github/workflows/check_deps_integrity.yaml   |   2 +-
 .github/workflows/code_style_check.yaml       |   2 +-
 .github/workflows/elixir_apps_check.yaml      |   2 +-
 .github/workflows/elixir_deps_check.yaml      |   2 +-
 .github/workflows/elixir_release.yml          |   2 +-
 .github/workflows/run_emqx_app_tests.yaml     |   2 +-
 .github/workflows/run_fvt_tests.yaml          |   6 +-
 .github/workflows/run_relup_tests.yaml        | 201 +++++++++++-------
 .github/workflows/run_test_cases.yaml         |   8 +-
 Makefile                                      |   2 +-
 deploy/docker/Dockerfile                      |   2 +-
 scripts/buildx.sh                             |   4 +-
 17 files changed, 155 insertions(+), 124 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index 83b58b0b9..2612eb8d8 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang24:
     container_name: erlang24
-    image: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
     env_file:
       - conf.env
     environment:
diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index 57afce294..06101ceb4 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -1,14 +1,11 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
-[config var=BENCH_PATH]
 [config var=VSN]
-[config var=OLD_VSNS]
+[config var=OLD_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
 
-[loop old_vsn $OLD_VSNS]
-
 [shell http_server]
     !cd http_server
     !rebar3 shell
@@ -22,10 +19,10 @@
 [shell emqx1]
     !cd $PACKAGE_PATH
     !mkdir -p emqx1
-    !tar -C emqx -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !tar -C emqx1 -zxf ${PROFILE}-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
     ?SH-PROMPT
 
-    !cd emqx
+    !cd emqx1
     !export EMQX_NODE_NAME='emqx1@127.0.0.1'
     !./bin/emqx start
     ?EMQX .* is started successfully!
@@ -34,7 +31,7 @@
 [shell emqx2]
     !cd $PACKAGE_PATH
     !mkdir -p emqx2
-    !tar -C emqx -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !tar -C emqx2 -zxf ${PROFILE}-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
     ?SH-PROMPT
 
     !cd emqx2
@@ -58,12 +55,12 @@
     ?SH-PROMPT
 
     ## create a webhook data bridge with id "my_webhook"
-    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","method":"post","url":"http://127.0.0.1:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status'
+    !curl --user admin:public --silent --show-error 'http://localhost:18082/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","method":"post","url":"http://127.0.0.1:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status'
     ?connected
     ?SH-PROMPT
 
     ## create a rule that uses the webhook as action, the rule id = "rule_edsy"
-    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n  *\nFROM\n  \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id'
+    !curl --user admin:public --silent --show-error 'http://localhost:18082/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n  *\nFROM\n  \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id'
     ?rule_edsy
     ?SH-PROMPT
 
@@ -77,9 +74,8 @@
     ?SH-PROMPT
 
 [shell bench]
-    !cd $BENCH_PATH
 
-    !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
+    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx1]
@@ -98,11 +94,11 @@
     ?SH-PROMPT
 
     ## 2. downgrade to the old version
-    !./bin/emqx install $old_vsn
+    !./bin/emqx install $OLD_VSN
     ?Made release permanent:.*
     ?SH-PROMPT
 
-    !./bin/emqx versions | grep permanent | grep -qs "$old_vsn"
+    !./bin/emqx versions | grep permanent | grep -qs "$OLD_VSN"
     ?SH-PROMPT:
     !echo ==$$?==
     ?^==0==
@@ -137,11 +133,11 @@
     ?SH-PROMPT
 
     ## 2. downgrade to the old version
-    !./bin/emqx install $old_vsn
+    !./bin/emqx install $OLD_VSN
     ?Made release permanent:.*
     ?SH-PROMPT
 
-    !./bin/emqx versions | grep permanent | grep -qs "$old_vsn"
+    !./bin/emqx versions | grep permanent | grep -qs "$OLD_VSN"
     ?SH-PROMPT:
     !echo ==$$?==
     ?^==0==
@@ -168,11 +164,11 @@
     !sleep 5
     ?SH-PROMPT
 
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.matched"
+    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx1@127.0.0.1\") | .metrics.matched"
     ?300
     ?SH-PROMPT
 
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.\"actions.success\""
+    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx1@127.0.0.1\") | .metrics.\"actions.success\""
     ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
@@ -212,8 +208,6 @@
     !halt(3).
     ?SH-PROMPT:
 
-[endloop]
-
 [cleanup]
     !echo ==$$?==
     ?==0==
diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index 92ec10f49..c6b9c5a8d 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -20,7 +20,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
 
     outputs:
       BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
@@ -211,7 +211,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
           RUN_FROM=${{ matrix.os[1] }}
           EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }}
         file: source/${{ matrix.os[2] }}
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index f99184002..2f378bfa2 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -213,7 +213,7 @@ jobs:
     needs: prepare
     runs-on: ${{ matrix.build_machine }}
     container:
-      image: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+      image: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     strategy:
       fail-fast: false
@@ -328,7 +328,7 @@ jobs:
             --pkgtype "${PKGTYPE}" \
             --arch "${ARCH}" \
             --elixir "${IsElixir}" \
-            --builder "ghcr.io/emqx/emqx-builder/5.0-16:${ELIXIR}-${OTP}-${SYSTEM}"
+            --builder "ghcr.io/emqx/emqx-builder/5.0-17:${ELIXIR}-${OTP}-${SYSTEM}"
         done
     - uses: actions/upload-artifact@v1
       with:
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 6cfcb5ee0..ba27c1b38 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -39,7 +39,7 @@ jobs:
         - ubuntu20.04
         - el8
 
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     steps:
     - uses: AutoModality/action-clean@v1
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index f696406af..4a6c31b5e 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request, push]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml
index 973312d76..5fbf91236 100644
--- a/.github/workflows/code_style_check.yaml
+++ b/.github/workflows/code_style_check.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   code_style_check:
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
     steps:
       - uses: actions/checkout@v2
         with:
diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml
index 524943d43..440c91545 100644
--- a/.github/workflows/elixir_apps_check.yaml
+++ b/.github/workflows/elixir_apps_check.yaml
@@ -8,7 +8,7 @@ jobs:
   elixir_apps_check:
     runs-on: ubuntu-latest
     # just use the latest builder
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
 
     strategy:
       fail-fast: false
diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml
index 49d16d4bf..312278caa 100644
--- a/.github/workflows/elixir_deps_check.yaml
+++ b/.github/workflows/elixir_deps_check.yaml
@@ -7,7 +7,7 @@ on: [pull_request, push]
 jobs:
   elixir_deps_check:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml
index e2380c21f..006d6aba8 100644
--- a/.github/workflows/elixir_release.yml
+++ b/.github/workflows/elixir_release.yml
@@ -12,7 +12,7 @@ on:
 jobs:
   elixir_release_build:
     runs-on: ubuntu-latest
-    container: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml
index 7539d6d8b..b2f13e8be 100644
--- a/.github/workflows/run_emqx_app_tests.yaml
+++ b/.github/workflows/run_emqx_app_tests.yaml
@@ -24,7 +24,7 @@ jobs:
           - amd64
 
     runs-on: aws-amd64
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
 
     defaults:
       run:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 2703630f2..8d9d65d2e 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -16,7 +16,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-alpine3.15.1
+    container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-alpine3.15.1
 
     steps:
       - uses: actions/checkout@v2
@@ -68,7 +68,7 @@ jobs:
     - name: make docker image
       working-directory: source
       env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
         EMQX_RUNNER: ${{ matrix.os[1] }}
       run: |
         make ${{ matrix.profile }}-docker
@@ -140,7 +140,7 @@ jobs:
     - name: make docker image
       working-directory: source
       env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
         EMQX_RUNNER: ${{ matrix.os[1] }}
       run: |
         make ${{ matrix.profile }}-docker
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 15c3d02ea..89c9cf8e4 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -14,104 +14,141 @@ on:
   pull_request:
 
 jobs:
-  relup_test:
-    runs-on: aws-amd64
-
-    strategy:
-      matrix:
-        profile:
-          - emqx
-          - emqx-enterprise
-        otp:
-          - 24.2.1-1
-        # no need to use more than 1 version of Elixir, since tests
-        # run using only Erlang code.  This is needed just to specify
-        # the base image.
-        elixir:
-          - 1.13.4
-        os:
-          - ubuntu20.04
-        arch:
-          - amd64
-
-    container: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
-
+  relup_test_plan:
+    runs-on: ubuntu-20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04
+    outputs:
+      ce_vsn: ${{ steps.find-versions.outputs.CUR_CE_VSN }}
+      ee_vsn: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
+      old_vsns: ${{ steps.generate-matrix.outputs.OLD_VERSIONS }}
     defaults:
       run:
         shell: bash
     steps:
-    - uses: AutoModality/action-clean@v1
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-        architecture: 'x64'
     - uses: actions/checkout@v2
+      name: Checkout
       with:
-        repository: emqx/paho.mqtt.testing
-        ref: develop-4.0
-        path: paho.mqtt.testing
-    - uses: actions/checkout@v2
-      with:
-        repository: emqx/emqtt-bench
-        ref: 0.3.4
-        path: emqtt-bench
-    - uses: actions/checkout@v2
-      with:
-        repository: hawk/lux
-        ref: lux-2.6
-        path: lux
-    - uses: actions/checkout@v2
-      with:
-        repository: ${{ github.repository }}
         path: emqx
         fetch-depth: 0
-
-    - name: Get old vsn
+    - name: Find versions
+      id: find-versions
       run: |
         set -x
-        echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
-        echo "NOW_VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
+        cd emqx
+        ce_vsn="$(./pkg-vsn.sh opensource)"
+        ee_vsn="$(./pkg-vsn.sh enterprise)"
+        old_ce_vsns="$(./scripts/relup-base-vsns.sh opensource | xargs)"
+        old_ee_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
+        echo "OLD_CE_VERSIONS=$old_ce_vsns" >> $GITHUB_ENV
+        echo "OLD_EE_VERSIONS=$old_ee_vsns" >> $GITHUB_ENV
+        echo "::set-output name=CUR_CE_VSN::$ce_vsn"
+        echo "::set-output name=CUR_EE_VSN::$ee_vsn"
+    - name: Generate matrix
+      id: generate-matrix
+      run: |
+        set -x
+        old_vsns=$(echo -n "${OLD_CE_VSNS} ${OLD_EE_VSNS}" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
+        echo "::set-output name=OLD_VERSIONS::$old_vsns"
 
+  relup_test_build:
+    needs: relup_test_plan
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        profile:
+        - emqx
+        - emqx-enterprise
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
+    defaults:
+      run:
+        shell: bash
+    steps:
     - name: build emqx
       env:
         PROFILE: ${{ matrix.profile }}
       run: make -C emqx ${PROFILE}-tgz
-    - name: build emqtt-bench
-      run: make -C emqtt-bench
-    - name: build lux
-      run: |
-        set -e -u -x
-        cd lux
-        autoconf
-        ./configure
-        make
-        make install
-    - name: run relup test
-      env:
-        PROFILE: ${{ matrix.profile }}
-      timeout-minutes: 20
+    - uses: actions/upload-artifact@v2
+      name: Upload built emqx and test scenario
+      with:
+        name: emqx_built
+        path: |
+          emqx/_packages/*/*.tar.gz
+          emqx/.ci/fvt_tests
+
+  relup_test_run:
+    needs:
+      - relup_test_plan
+      - relup_test_build
+    runs-on: ubuntu-20.04
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
+    strategy:
+      fail-fast: false
+      matrix:
+        old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.OLD_VERSIONS) }}
+    env:
+      OLD_VSN: "${{ matrix.old_vsn }}"
+      CUR_CE_VSN: "${{ needs.relup_test_plan.outputs.CUR_CE_VSN }}"
+      CUR_EE_VSN: "${{ needs.relup_test_plan.outputs.CUR_EE_VSN }}"
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/download-artifact@v2
+      name: Download built emqx and test scenario
+      with:
+        name: emqx_built
+        path: emqx_built
+    - name: Prepare packages
       run: |
         set -e -x -u
-        if [ -n "$OLD_VSNS" ]; then
-            mkdir -p packages
-            cp emqx/_packages/${PROFILE}/*.tar.gz packages/
-            cp emqx/_upgrade_base/*.tar.gz packages/
-            lux \
-            --case_timeout infinity \
-            --var PROFILE=$PROFILE \
-            --var PACKAGE_PATH=$(pwd)/packages \
-            --var BENCH_PATH=$(pwd)/emqtt-bench \
-            --var VSN="$NOW_VSN" \
-            --var OLD_VSNS="$OLD_VSNS" \
-            emqx/.ci/fvt_tests/relup.lux
-        fi
-    - uses: actions/upload-artifact@v1
+        mkdir -p packages
+        cp emqx_built/_packages/*/*.tar.gz packages
+        cd packages
+        case "$OLD_VSN" in
+          e*)
+            profile='emqx-enterprise'
+            s3dir='emqx-ee'
+            ;;
+          v*)
+            profile='emqx'
+            s3dir='emqx-ce'
+            ;;
+          *)
+            echo "unknown old version $OLD_VSN"
+            exit 1
+            ;;
+        esac
+        wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/${s3dir}/$OLD_VSN/${profile}-${OLD_VSN#[e|v]}-24.2.1-1-ubuntu20.04-amd64.tar.gz
+    - name: Run relup test scenario
+      timeout-minutes: 5
+      run: |
+        set -x
+        case "$OLD_VSN" in
+          e*)
+            cur_vsn=$CUR_EE_VSN
+            profile='emqx-enterprise'
+            ;;
+          v*)
+            cur_vsn=$CUR_CE_VSN
+            profile='emqx'
+            ;;
+        esac
+
+        lux \
+        --progress verbose \
+        --case_timeout infinity \
+        --var PROFILE="$profile" \
+        --var PACKAGE_PATH=$(pwd)/packages \
+        --var VSN="$cur_vsn" \
+        --var OLD_VSN="$OLD_VSN" \
+        emqx_built/.ci/fvt_tests/relup.lux
+    - uses: actions/upload-artifact@v2
+      name: Save debug data
       if: failure()
       with:
-        name: lux_logs
-        path: lux_logs
-    - uses: actions/upload-artifact@v1
-      if: failure()
-      with:
-        name: packages
-        path: packages
+        name: debug_data
+        path: |
+          packages/emqx1/log/emqx.log.1
+          packages/emqx2/log/emqx.log.1
+          packages/*.zip
+          lux_logs
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 1d7c7a979..d38acaf31 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -17,7 +17,7 @@ jobs:
     prepare:
         runs-on: ubuntu-20.04
         # prepare source with any OTP version, no need for a matrix
-        container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+        container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
         outputs:
           fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
           docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
@@ -54,7 +54,7 @@ jobs:
         defaults:
           run:
             shell: bash
-        container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+        container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
 
         steps:
         - uses: AutoModality/action-clean@v1
@@ -144,7 +144,7 @@ jobs:
           matrix:
             app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
         runs-on: aws-amd64
-        container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+        container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
         defaults:
           run:
             shell: bash
@@ -179,7 +179,7 @@ jobs:
         - ct
         - ct_docker
       runs-on: ubuntu-20.04
-      container: "ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-ubuntu20.04"
+      container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
       steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v2
diff --git a/Makefile b/Makefile
index ceb5a9934..6be3f76c9 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-debian11
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 5213579a7..0fb259082 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-debian11
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11
 ARG RUN_FROM=debian:11-slim
 FROM ${BUILD_FROM} AS builder
 
diff --git a/scripts/buildx.sh b/scripts/buildx.sh
index c87ea44d2..96e48d9eb 100755
--- a/scripts/buildx.sh
+++ b/scripts/buildx.sh
@@ -9,7 +9,7 @@
 
 ## example:
 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
-##     --builder ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-debian10
+##     --builder ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10
 
 set -euo pipefail
 
@@ -24,7 +24,7 @@ help() {
     echo "--arch amd64|arm64:        Target arch to build the EMQX package for"
     echo "--src_dir <SRC_DIR>:       EMQX source ode in this dir, default to PWD"
     echo "--builder <BUILDER>:       Builder image to pull"
-    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-debian10"
+    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10"
 }
 
 while [ "$#" -gt 0 ]; do

From 1ba3ae9a44d928d64dbc14b466ae69225e2159c3 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 15:05:00 +0200
Subject: [PATCH 09/23] chore: build emqx packages in one go

---
 .github/workflows/run_relup_tests.yaml | 33 ++++++--------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 89c9cf8e4..27abe3c65 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -16,7 +16,7 @@ on:
 jobs:
   relup_test_plan:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04
+    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
     outputs:
       ce_vsn: ${{ steps.find-versions.outputs.CUR_CE_VSN }}
       ee_vsn: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
@@ -39,34 +39,16 @@ jobs:
         ee_vsn="$(./pkg-vsn.sh enterprise)"
         old_ce_vsns="$(./scripts/relup-base-vsns.sh opensource | xargs)"
         old_ee_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
-        echo "OLD_CE_VERSIONS=$old_ce_vsns" >> $GITHUB_ENV
-        echo "OLD_EE_VERSIONS=$old_ee_vsns" >> $GITHUB_ENV
+        old_vsns=$(echo -n "${old_ce_vsns} ${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
         echo "::set-output name=CUR_CE_VSN::$ce_vsn"
         echo "::set-output name=CUR_EE_VSN::$ee_vsn"
-    - name: Generate matrix
-      id: generate-matrix
+        echo "::set-output name=OLD_VERSIONS::$old_vsns"
+    - name: build emqx
       run: |
         set -x
-        old_vsns=$(echo -n "${OLD_CE_VSNS} ${OLD_EE_VSNS}" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
-        echo "::set-output name=OLD_VERSIONS::$old_vsns"
-
-  relup_test_build:
-    needs: relup_test_plan
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        profile:
-        - emqx
-        - emqx-enterprise
-    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
-    defaults:
-      run:
-        shell: bash
-    steps:
-    - name: build emqx
-      env:
-        PROFILE: ${{ matrix.profile }}
-      run: make -C emqx ${PROFILE}-tgz
+        cd emqx
+        make emqx-tgz
+        make emqx-enterprise-tgz
     - uses: actions/upload-artifact@v2
       name: Upload built emqx and test scenario
       with:
@@ -78,7 +60,6 @@ jobs:
   relup_test_run:
     needs:
       - relup_test_plan
-      - relup_test_build
     runs-on: ubuntu-20.04
     container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
     strategy:

From 129915a199ab5659d9c2acaddde95d1a19a43233 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 15:20:34 +0200
Subject: [PATCH 10/23] ci: fix build output name

---
 .github/workflows/run_relup_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 27abe3c65..a30484ed6 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -20,7 +20,7 @@ jobs:
     outputs:
       ce_vsn: ${{ steps.find-versions.outputs.CUR_CE_VSN }}
       ee_vsn: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
-      old_vsns: ${{ steps.generate-matrix.outputs.OLD_VERSIONS }}
+      old_vsns: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
     defaults:
       run:
         shell: bash

From 46e5127b15ad1a7056684d0038aa388dd8c92276 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 15:49:27 +0200
Subject: [PATCH 11/23] ci: fix type in ci job output

---
 .github/workflows/run_relup_tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index a30484ed6..626bd48aa 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -18,9 +18,9 @@ jobs:
     runs-on: ubuntu-20.04
     container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
     outputs:
-      ce_vsn: ${{ steps.find-versions.outputs.CUR_CE_VSN }}
-      ee_vsn: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
-      old_vsns: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
+      CUR_CE_VSN: ${{ steps.find-versions.outputs.CUR_CE_VSN }}
+      CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
+      OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
     defaults:
       run:
         shell: bash

From 4ecff7bd32710fc1294b061ec9d765853a84f75b Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 19:42:43 +0200
Subject: [PATCH 12/23] ci: fix base package download url

---
 .github/workflows/run_relup_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 626bd48aa..9095d4be0 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -99,7 +99,7 @@ jobs:
             exit 1
             ;;
         esac
-        wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/${s3dir}/$OLD_VSN/${profile}-${OLD_VSN#[e|v]}-24.2.1-1-ubuntu20.04-amd64.tar.gz
+        wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/${s3dir}/$OLD_VSN/${profile}-${OLD_VSN#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz
     - name: Run relup test scenario
       timeout-minutes: 5
       run: |

From 5c7298eb4129b772cc6e322851a26bc141770a4a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 20:06:49 +0200
Subject: [PATCH 13/23] ci: fix wrong upload dir in case of relup test falure

---
 .github/workflows/run_relup_tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 9095d4be0..8516b005b 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -129,7 +129,7 @@ jobs:
       with:
         name: debug_data
         path: |
-          packages/emqx1/log/emqx.log.1
-          packages/emqx2/log/emqx.log.1
-          packages/*.zip
+          packages/emqx1/log/*
+          packages/emqx2/log/*
+          packages/*.tar.gz
           lux_logs

From 67710472f57f416144f7edfcad8b24299da89e22 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 20:49:20 +0200
Subject: [PATCH 14/23] ci: debug boot in relup test

---
 .ci/fvt_tests/relup.lux | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index 06101ceb4..e631a7d0b 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -24,6 +24,8 @@
 
     !cd emqx1
     !export EMQX_NODE_NAME='emqx1@127.0.0.1'
+# debug boot
+    !sed 's/-boot_var RELEASE_LIB/-init_debug -boot_var RELEASE_LIB/g' -i ./bin/emqx
     !./bin/emqx start
     ?EMQX .* is started successfully!
     ?SH-PROMPT

From f7a2112a5ab04e4068bde776cfec91a9ddade7ff Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Jun 2022 22:06:18 +0200
Subject: [PATCH 15/23] ci: define package name instead of globing

---
 .ci/fvt_tests/relup.lux                | 10 ++++++----
 .github/workflows/run_relup_tests.yaml | 10 ++++++----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index e631a7d0b..83937291a 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -1,7 +1,9 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
 [config var=VSN]
+[config var=CUR_PKG]
 [config var=OLD_VSN]
+[config var=OLD_PKG]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
@@ -19,7 +21,7 @@
 [shell emqx1]
     !cd $PACKAGE_PATH
     !mkdir -p emqx1
-    !tar -C emqx1 -zxf ${PROFILE}-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !tar -C emqx1 -zxf "$OLD_PKG"
     ?SH-PROMPT
 
     !cd emqx1
@@ -33,7 +35,7 @@
 [shell emqx2]
     !cd $PACKAGE_PATH
     !mkdir -p emqx2
-    !tar -C emqx2 -zxf ${PROFILE}-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
+    !tar -C emqx2 -zxf "$OLD_PKG"
     ?SH-PROMPT
 
     !cd emqx2
@@ -84,7 +86,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
+    !cp -f ../$CUR_PKG releases/
 
     ## 1. upgrade to the new version
     !./bin/emqx install $VSN
@@ -123,7 +125,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
+    !cp -f ../$CUR_PKG releases/
 
     ## 1. upgrade to the new version
     !./bin/emqx install $VSN
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 8516b005b..061ca3d20 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -114,7 +114,8 @@ jobs:
             profile='emqx'
             ;;
         esac
-
+        old_pkg="${profile}-${OLD_VSN#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
+        cur_pkg="${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
         lux \
         --progress verbose \
         --case_timeout infinity \
@@ -122,6 +123,8 @@ jobs:
         --var PACKAGE_PATH=$(pwd)/packages \
         --var VSN="$cur_vsn" \
         --var OLD_VSN="$OLD_VSN" \
+        --var CUR_PKG="$cur_pkg" \
+        --var OLD_PKG="$old_pkg" \
         emqx_built/.ci/fvt_tests/relup.lux
     - uses: actions/upload-artifact@v2
       name: Save debug data
@@ -129,7 +132,6 @@ jobs:
       with:
         name: debug_data
         path: |
-          packages/emqx1/log/*
-          packages/emqx2/log/*
-          packages/*.tar.gz
+          packages/emqx1/*
+          packages/emqx2/*
           lux_logs

From 184b337e8477d53cac4dcfb8ad2a9e876e1b9508 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 28 Jun 2022 10:54:07 +0200
Subject: [PATCH 16/23] chore: add a script to run relup lux test locally

---
 .ci/fvt_tests/relup.lux                | 16 +++++-----
 .github/workflows/run_relup_tests.yaml |  2 +-
 scripts/run-relup-lux.sh               | 42 ++++++++++++++++++++++++++
 scripts/start-two-nodes-in-docker.sh   |  2 +-
 4 files changed, 51 insertions(+), 11 deletions(-)
 create mode 100755 scripts/run-relup-lux.sh

diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index 83937291a..de48a723a 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -1,5 +1,5 @@
+[config var=PROJ_ROOT]
 [config var=PROFILE]
-[config var=PACKAGE_PATH]
 [config var=VSN]
 [config var=CUR_PKG]
 [config var=OLD_VSN]
@@ -19,21 +19,19 @@
     ?>
 
 [shell emqx1]
-    !cd $PACKAGE_PATH
+    !cd $PROJ_ROOT
     !mkdir -p emqx1
     !tar -C emqx1 -zxf "$OLD_PKG"
     ?SH-PROMPT
 
     !cd emqx1
     !export EMQX_NODE_NAME='emqx1@127.0.0.1'
-# debug boot
-    !sed 's/-boot_var RELEASE_LIB/-init_debug -boot_var RELEASE_LIB/g' -i ./bin/emqx
     !./bin/emqx start
     ?EMQX .* is started successfully!
     ?SH-PROMPT
 
 [shell emqx2]
-    !cd $PACKAGE_PATH
+    !cd $PROJ_ROOT
     !mkdir -p emqx2
     !tar -C emqx2 -zxf "$OLD_PKG"
     ?SH-PROMPT
@@ -182,7 +180,7 @@
     ?SH-PROMPT
 
 [shell emqx2]
-    !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
+    !cat log/emqx.log.1 | tail -n 100
     -error
     ??SH-PROMPT:
 
@@ -190,11 +188,11 @@
     ?ok
     ?SH-PROMPT:
 
-    !rm -rf $PACKAGE_PATH/emqx2
+    !rm -rf emqx2/
     ?SH-PROMPT:
 
 [shell emqx1]
-    !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
+    !cat log/emqx.log.1 | tail -n 100
     -error
     ??SH-PROMPT:
 
@@ -202,7 +200,7 @@
     ?ok
     ?SH-PROMPT:
 
-    !rm -rf $PACKAGE_PATH/emqx
+    !rm -rf emqx1/
     ?SH-PROMPT:
 
 [shell http_server]
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 061ca3d20..d5c140763 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -119,8 +119,8 @@ jobs:
         lux \
         --progress verbose \
         --case_timeout infinity \
+        --var PROJ_ROOT="$(pwd)" \
         --var PROFILE="$profile" \
-        --var PACKAGE_PATH=$(pwd)/packages \
         --var VSN="$cur_vsn" \
         --var OLD_VSN="$OLD_VSN" \
         --var CUR_PKG="$cur_pkg" \
diff --git a/scripts/run-relup-lux.sh b/scripts/run-relup-lux.sh
new file mode 100755
index 000000000..5b859c2c0
--- /dev/null
+++ b/scripts/run-relup-lux.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+## This script needs the 'lux' command in PATH
+## it runs the .ci/fvt_tests/relup.lux script
+
+set -euo pipefail
+
+old_vsn="${1}"
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")/.."
+
+set -x
+
+case "$old_vsn" in
+    e*)
+        cur_vsn="$(./pkg-vsn.sh emqx-enterprise)"
+        profile='emqx-enterprise'
+        ;;
+    v*)
+        cur_vsn="$(./pkg-vsn.sh emqx)"
+        profile='emqx'
+        ;;
+    *)
+        echo "unknown old version $old_vsn"
+        exit 1
+        ;;
+esac
+
+old_pkg="$(pwd)/_upgrade_base/${profile}-${old_vsn#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
+cur_pkg="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
+
+lux \
+    --progress verbose \
+    --case_timeout infinity \
+    --var PROJ_ROOT="$(pwd)" \
+    --var PROFILE="$profile" \
+    --var VSN="$cur_vsn" \
+    --var OLD_VSN="$old_vsn" \
+    --var CUR_PKG="$cur_pkg" \
+    --var OLD_PKG="$old_pkg" \
+    .ci/fvt_tests/relup.lux
diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/start-two-nodes-in-docker.sh
index 64a6647ce..27ec4a1b2 100755
--- a/scripts/start-two-nodes-in-docker.sh
+++ b/scripts/start-two-nodes-in-docker.sh
@@ -40,7 +40,7 @@ docker run -d -t --restart=always --name "$NODE2" \
   -p 18084:18083 \
   "$IMAGE"
 
-wait (){
+wait () {
   container="$1"
   while ! docker exec "$container" emqx_ctl status >/dev/null 2>&1; do
     echo -n '.'

From 8b7fc490aeb674e4720c8d2976196ed35839d93d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 28 Jun 2022 22:06:00 +0200
Subject: [PATCH 17/23] refactor: run relup test nodes in individual docker
 containers

When running EMQX in lux shell inside a docker container
the node won't boot due to load_failed for redbug modules
reason is still unknown
---
 .ci/fvt_tests/http_server/src/http_server.erl |  12 +-
 .ci/fvt_tests/relup.lux                       | 215 ------------------
 .github/workflows/run_relup_tests.yaml        |  79 +++----
 .gitignore                                    |   1 +
 scripts/relup/check-results.sh                |  41 ++++
 scripts/relup/relup.lux                       | 130 +++++++++++
 scripts/relup/run-pkg.sh                      |  16 ++
 scripts/relup/run-relup-lux.sh                |  64 ++++++
 scripts/relup/start-relup-test-cluster.sh     | 111 +++++++++
 scripts/run-relup-lux.sh                      |  42 ----
 10 files changed, 400 insertions(+), 311 deletions(-)
 delete mode 100644 .ci/fvt_tests/relup.lux
 create mode 100755 scripts/relup/check-results.sh
 create mode 100644 scripts/relup/relup.lux
 create mode 100755 scripts/relup/run-pkg.sh
 create mode 100755 scripts/relup/run-relup-lux.sh
 create mode 100755 scripts/relup/start-relup-test-cluster.sh
 delete mode 100755 scripts/run-relup-lux.sh

diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl
index 4b1981de4..4aaa25b95 100644
--- a/.ci/fvt_tests/http_server/src/http_server.erl
+++ b/.ci/fvt_tests/http_server/src/http_server.erl
@@ -32,7 +32,7 @@
 
 start() ->
     application:ensure_all_started(minirest),
-    ets:new(relup_test_message, [named_table, public]),
+    _ = spawn(fun ets_owner/0),
     Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}],
     Dispatch = [{"/[...]", minirest, Handlers}],
     minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 7077}]}, Dispatch).
@@ -42,7 +42,8 @@ stop() ->
     minirest:stop_http(?MODULE).
 
 get_counter(_Binding, _Params) ->
-    return({ok, ets:info(relup_test_message, size)}).
+    V = ets:info(relup_test_message, size),
+    return({ok, V}).
 
 add_counter(_Binding, Params) ->
     case lists:keymember(<<"payload">>, 1, Params) of
@@ -50,6 +51,13 @@ add_counter(_Binding, Params) ->
             {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params),
             ets:insert(relup_test_message, {ID, Params1});
         _ ->
+            io:format("discarded: ~p\n", [Params]),
             ok
     end,
     return().
+
+ets_owner() ->
+    ets:new(relup_test_message, [named_table, public]),
+    receive
+        stop -> ok
+    end.
diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
deleted file mode 100644
index de48a723a..000000000
--- a/.ci/fvt_tests/relup.lux
+++ /dev/null
@@ -1,215 +0,0 @@
-[config var=PROJ_ROOT]
-[config var=PROFILE]
-[config var=VSN]
-[config var=CUR_PKG]
-[config var=OLD_VSN]
-[config var=OLD_PKG]
-
-[config shell_cmd=/bin/bash]
-[config timeout=600000]
-
-[shell http_server]
-    !cd http_server
-    !rebar3 shell
-    ???Eshell
-    ???>
-    !http_server:start().
-    ?Start http_server listener on 7077 successfully.
-    ?ok
-    ?>
-
-[shell emqx1]
-    !cd $PROJ_ROOT
-    !mkdir -p emqx1
-    !tar -C emqx1 -zxf "$OLD_PKG"
-    ?SH-PROMPT
-
-    !cd emqx1
-    !export EMQX_NODE_NAME='emqx1@127.0.0.1'
-    !./bin/emqx start
-    ?EMQX .* is started successfully!
-    ?SH-PROMPT
-
-[shell emqx2]
-    !cd $PROJ_ROOT
-    !mkdir -p emqx2
-    !tar -C emqx2 -zxf "$OLD_PKG"
-    ?SH-PROMPT
-
-    !cd emqx2
-    !export EMQX_NODE__NAME='emqx2@127.0.0.1'
-    !export EMQX_STATSD__SERVER='127.0.0.1:8124'
-    !export EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1882'
-    !export EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8882'
-    !export EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8082'
-    !export EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8085'
-    !export EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18082'
-    !./bin/emqx start
-    ?EMQX .* is started successfully!
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl cluster join emqx1@127.0.0.1
-    ???Join the cluster successfully.
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
-    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
-    ?SH-PROMPT
-
-    ## create a webhook data bridge with id "my_webhook"
-    !curl --user admin:public --silent --show-error 'http://localhost:18082/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","method":"post","url":"http://127.0.0.1:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status'
-    ?connected
-    ?SH-PROMPT
-
-    ## create a rule that uses the webhook as action, the rule id = "rule_edsy"
-    !curl --user admin:public --silent --show-error 'http://localhost:18082/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n  *\nFROM\n  \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id'
-    ?rule_edsy
-    ?SH-PROMPT
-
-[shell emqx1]
-    ## verify the bridges and rules are sync to the other node
-    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges/webhook:my_webhook' -X 'GET' -H 'Content-Type: application/json' | jq '.name'
-    ?my_webhook
-    ?SH-PROMPT
-    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules/rule_edsy' -X 'GET' -H 'Content-Type: application/json' | jq '.id'
-    ?rule_edsy
-    ?SH-PROMPT
-
-[shell bench]
-
-    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
-    ???sent
-
-[shell emqx1]
-    !echo "" > log/emqx.log.1
-    ?SH-PROMPT
-
-    !cp -f ../$CUR_PKG releases/
-
-    ## 1. upgrade to the new version
-    !./bin/emqx install $VSN
-    ?Made release permanent: "$VSN"
-    ?SH-PROMPT
-
-    !./bin/emqx versions | grep permanent
-    ?(.*)$VSN
-    ?SH-PROMPT
-
-    ## 2. downgrade to the old version
-    !./bin/emqx install $OLD_VSN
-    ?Made release permanent:.*
-    ?SH-PROMPT
-
-    !./bin/emqx versions | grep permanent | grep -qs "$OLD_VSN"
-    ?SH-PROMPT:
-    !echo ==$$?==
-    ?^==0==
-    ?SH-PROMPT:
-
-    ## 3. again, upgrade to the new version
-    !./bin/emqx install $VSN
-    ?Made release permanent: "$VSN"
-    ?SH-PROMPT
-
-    !./bin/emqx versions | grep permanent
-    ?(.*)$VSN
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
-    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
-    ?SH-PROMPT
-
-[shell emqx2]
-    !echo "" > log/emqx.log.1
-    ?SH-PROMPT
-
-    !cp -f ../$CUR_PKG releases/
-
-    ## 1. upgrade to the new version
-    !./bin/emqx install $VSN
-    ?Made release permanent: "$VSN"
-    ?SH-PROMPT
-
-    !./bin/emqx versions |grep permanent
-    ?(.*)$VSN
-    ?SH-PROMPT
-
-    ## 2. downgrade to the old version
-    !./bin/emqx install $OLD_VSN
-    ?Made release permanent:.*
-    ?SH-PROMPT
-
-    !./bin/emqx versions | grep permanent | grep -qs "$OLD_VSN"
-    ?SH-PROMPT:
-    !echo ==$$?==
-    ?^==0==
-    ?SH-PROMPT:
-
-    ## 3. again, upgrade to the new version
-    !./bin/emqx install $VSN
-    ?Made release permanent: "$VSN"
-    ?SH-PROMPT
-
-    !./bin/emqx versions |grep permanent
-    ?(.*)$VSN
-    ?SH-PROMPT
-
-    !./bin/emqx_ctl cluster status --json | jq -c .running_nodes
-    ???["emqx1@127.0.0.1","emqx2@127.0.0.1"]
-    ?SH-PROMPT
-
-## We don't guarantee not to lose a single message!
-## So even if we received 290~300 messages, we consider it as success
-[shell bench]
-    ???publish complete
-    ??SH-PROMPT:
-    !sleep 5
-    ?SH-PROMPT
-
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx1@127.0.0.1\") | .metrics.matched"
-    ?300
-    ?SH-PROMPT
-
-    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx1@127.0.0.1\") | .metrics.\"actions.success\""
-    ?\{"data":(29[0-9])|(300),"code":0\}
-    ?SH-PROMPT
-
-    ## The /counter API is provided by .ci/fvt_test/http_server
-    !curl http://127.0.0.1:7077/counter
-    ?\{"data":(29[0-9])|(300),"code":0\}
-    ?SH-PROMPT
-
-[shell emqx2]
-    !cat log/emqx.log.1 | tail -n 100
-    -error
-    ??SH-PROMPT:
-
-    !./bin/emqx stop
-    ?ok
-    ?SH-PROMPT:
-
-    !rm -rf emqx2/
-    ?SH-PROMPT:
-
-[shell emqx1]
-    !cat log/emqx.log.1 | tail -n 100
-    -error
-    ??SH-PROMPT:
-
-    !./bin/emqx stop
-    ?ok
-    ?SH-PROMPT:
-
-    !rm -rf emqx1/
-    ?SH-PROMPT:
-
-[shell http_server]
-    !http_server:stop().
-    ?ok
-    ?>
-    !halt(3).
-    ?SH-PROMPT:
-
-[cleanup]
-    !echo ==$$?==
-    ?==0==
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index d5c140763..3d3223eb8 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -54,14 +54,12 @@ jobs:
       with:
         name: emqx_built
         path: |
-          emqx/_packages/*/*.tar.gz
-          emqx/.ci/fvt_tests
+          emqx/*
 
   relup_test_run:
     needs:
       - relup_test_plan
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
     strategy:
       fail-fast: false
       matrix:
@@ -74,64 +72,41 @@ jobs:
       run:
         shell: bash
     steps:
+      # setup Erlang to run lux
+    - uses: erlef/setup-beam@v1
+      with:
+        otp-version: "24.2"
+    - uses: actions/checkout@v2
+      with:
+        repository: hawk/lux
+        ref: lux-2.8.1
+        path: lux
+    - name: Install lux
+      run: |
+        set -e -u -x
+        cd lux
+        autoconf
+        ./configure
+        make
+        echo "$(pwd)/bin" >> $GITHUB_PATH
     - uses: actions/download-artifact@v2
       name: Download built emqx and test scenario
       with:
         name: emqx_built
-        path: emqx_built
-    - name: Prepare packages
+        path: .
+    - name: run relup test
       run: |
         set -e -x -u
-        mkdir -p packages
-        cp emqx_built/_packages/*/*.tar.gz packages
-        cd packages
-        case "$OLD_VSN" in
-          e*)
-            profile='emqx-enterprise'
-            s3dir='emqx-ee'
-            ;;
-          v*)
-            profile='emqx'
-            s3dir='emqx-ce'
-            ;;
-          *)
-            echo "unknown old version $OLD_VSN"
-            exit 1
-            ;;
-        esac
-        wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/${s3dir}/$OLD_VSN/${profile}-${OLD_VSN#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz
-    - name: Run relup test scenario
-      timeout-minutes: 5
-      run: |
-        set -x
-        case "$OLD_VSN" in
-          e*)
-            cur_vsn=$CUR_EE_VSN
-            profile='emqx-enterprise'
-            ;;
-          v*)
-            cur_vsn=$CUR_CE_VSN
-            profile='emqx'
-            ;;
-        esac
-        old_pkg="${profile}-${OLD_VSN#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
-        cur_pkg="${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
-        lux \
-        --progress verbose \
-        --case_timeout infinity \
-        --var PROJ_ROOT="$(pwd)" \
-        --var PROFILE="$profile" \
-        --var VSN="$cur_vsn" \
-        --var OLD_VSN="$OLD_VSN" \
-        --var CUR_PKG="$cur_pkg" \
-        --var OLD_PKG="$old_pkg" \
-        emqx_built/.ci/fvt_tests/relup.lux
+        cd emqx
+        if ! ./scripts/relup/run-relup-lux.sh $OLD_VSN; then
+          docker logs node1.emqx.io | tee lux_logs/emqx1.log
+          docker logs node2.emqx.io | tee lux_logs/emqx2.log
+          exit 1
+        fi
     - uses: actions/upload-artifact@v2
       name: Save debug data
       if: failure()
       with:
         name: debug_data
         path: |
-          packages/emqx1/*
-          packages/emqx2/*
-          lux_logs
+          wd/lux_logs
diff --git a/.gitignore b/.gitignore
index f4472b9ab..f0e55ad28 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,3 +66,4 @@ mix.lock
 apps/emqx/test/emqx_static_checks_data/master.bpapi
 # rendered configurations
 *.conf.rendered
+lux_logs/
diff --git a/scripts/relup/check-results.sh b/scripts/relup/check-results.sh
new file mode 100755
index 000000000..446ebeca5
--- /dev/null
+++ b/scripts/relup/check-results.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+matched_node1="$(curl --user admin:public -sf http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node1.emqx.io\") | .metrics.matched")"
+# TODO
+matched_node2=0
+#matched_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.matched")"
+success_node1="$(curl --user admin:public -sf http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node1.emqx.io\") | .metrics.\"actions.success\"")"
+# TODO
+success_node2=0
+#success_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.\"actions.success\"")"
+webhook="$(curl -sf http://localhost:7077/counter | jq '.data')"
+
+MATCHED_TOTAL="$(( matched_node1 + matched_node2 ))"
+SUCCESS_TOTAL="$(( success_node1 + success_node2 ))"
+COLLECTED_TOTAL="$webhook"
+
+is_number() {
+    re='^[0-9]+$'
+    if ! [[ $2 =~ $re ]] ; then
+       echo "error: $1=$2 is not a number" >&2; exit 1
+    fi
+}
+
+is_number MATCHED_TOTAL "$MATCHED_TOTAL"
+is_number SUCCESS_TOTAL "$SUCCESS_TOTAL"
+is_number COLLECTED_TOTAL "$COLLECTED_TOTAL"
+
+if [ "$MATCHED_TOTAL" -lt 290 ] || \
+   [ "$SUCCESS_TOTAL" -lt 290 ] || \
+   [ "$COLLECTED_TOTAL" -lt 290 ]; then
+    echo "FAILED"
+    echo "MATCHED_TOTAL=$MATCHED_TOTAL"
+    echo "SUCCESS_TOTAL=$SUCCESS_TOTAL"
+    echo "COLLECTED_TOTAL=$COLLECTED_TOTAL"
+    exit 1
+else
+    echo "ALL_IS_WELL"
+    exit 0
+fi
diff --git a/scripts/relup/relup.lux b/scripts/relup/relup.lux
new file mode 100644
index 000000000..88a504cb8
--- /dev/null
+++ b/scripts/relup/relup.lux
@@ -0,0 +1,130 @@
+[config var=PROJ_ROOT]
+[config var=VSN]
+[config var=CUR_PKG]
+[config var=OLD_VSN]
+[config var=NODE1]
+[config var=NODE2]
+[config var=BENCH]
+
+[config shell_cmd=/bin/bash]
+[config timeout=600000]
+
+[shell emqx1]
+    !docker exec -it $NODE1 emqx_ctl cluster status
+    ???running_nodes => ['emqx@node1.emqx.io','emqx@node2.emqx.io']
+    ?SH-PROMPT
+
+    ## create a webhook data bridge with id "my_webhook"
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","body":"","method":"post","url":"http://webhook.emqx.io:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status'
+    ?connected
+    ?SH-PROMPT
+
+    ## create a rule that uses the webhook as action, the rule id = "rule_edsy"
+    !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n  *\nFROM\n  \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id'
+    ?rule_edsy
+    ?SH-PROMPT
+
+[shell emqx2]
+    ## verify the bridges and rules are sync to the other node
+    !curl --user admin:public --silent --show-error 'http://localhost:18084/api/v5/bridges/webhook:my_webhook' -X 'GET' -H 'Content-Type: application/json' | jq '.name'
+    ?my_webhook
+    ?SH-PROMPT
+    !curl --user admin:public --silent --show-error 'http://localhost:18084/api/v5/rules/rule_edsy' -X 'GET' -H 'Content-Type: application/json' | jq '.id'
+    ?rule_edsy
+    ?SH-PROMPT
+
+[shell bench]
+    !docker exec -it $BENCH emqtt_bench pub --host 'node1.emqx.io' --port 1883 -c 10 -I 1000 -t t/%i -s 64 -L 300
+
+[shell emqx1]
+    !docker cp $CUR_PKG $NODE1:/emqx/releases/
+
+    ## 1. upgrade to the new version
+    !docker exec -it $NODE1 emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !docker exec -it $NODE1 emqx versions | grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## 2. downgrade to the old version
+    !docker exec -it $NODE1 emqx install $OLD_VSN
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !docker exec -it $NODE1 emqx versions | grep permanent | grep -qs "$OLD_VSN"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## 3. again, upgrade to the new version
+    !docker exec -it $NODE1 emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !docker exec -it $NODE1 emqx versions | grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    !docker exec -it $NODE1 emqx_ctl cluster status
+    ???running_nodes => ['emqx@node1.emqx.io','emqx@node2.emqx.io']
+    ?SH-PROMPT
+
+[shell emqx2]
+    !docker cp $CUR_PKG $NODE2:/emqx/releases/
+
+    ## 1. upgrade to the new version
+    !docker exec -it $NODE2 emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !docker exec -it $NODE2 emqx versions | grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## 2. downgrade to the old version
+    !docker exec -it $NODE2 emqx install $OLD_VSN
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !docker exec -it $NODE2 emqx versions | grep permanent | grep -qs "$OLD_VSN"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## 3. again, upgrade to the new version
+    !docker exec -it $NODE2 emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !docker exec -it $NODE2 emqx versions | grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    !docker exec -it $NODE1 emqx_ctl cluster status
+    ???running_nodes => ['emqx@node1.emqx.io','emqx@node2.emqx.io']
+    ?SH-PROMPT
+
+## We don't guarantee not to lose a single message!
+## So even if we received 290~300 messages, we consider it as success
+[shell bench]
+    ???publish complete
+    ??SH-PROMPT:
+    !sleep 5
+    ?SH-PROMPT
+
+    !$PROJ_ROOT/scripts/relup/check-results.sh
+    !echo ==$$?==
+    ???ALL_IS_WELL
+    ?SH-PROMPT:
+
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+[cleanup]
+    !echo ==$$?==
+    ?==0==
diff --git a/scripts/relup/run-pkg.sh b/scripts/relup/run-pkg.sh
new file mode 100755
index 000000000..0a7fda049
--- /dev/null
+++ b/scripts/relup/run-pkg.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+## This script is intended to run in docker
+## extracts a .tar.gz package and runs EMQX in console mode
+
+set -euo pipefail
+
+PKG="$1"
+
+mkdir -p emqx
+tar -C emqx -zxf "$PKG"
+
+ln -s "$(pwd)/emqx/bin/emqx" /usr/bin/emqx
+ln -s "$(pwd)/emqx/bin/emqx_ctl" /usr/bin/emqx_ctl
+
+emqx console
diff --git a/scripts/relup/run-relup-lux.sh b/scripts/relup/run-relup-lux.sh
new file mode 100755
index 000000000..af0ecf4c7
--- /dev/null
+++ b/scripts/relup/run-relup-lux.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+## This script needs the 'lux' command in PATH
+## it runs the scripts/relup/relup.lux script
+
+set -euo pipefail
+
+old_vsn="${1:-}"
+if [ -z "$old_vsn" ]; then
+    echo "arg1 should be the upgrade base version"
+    exit 1
+fi
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")/../.."
+
+set -x
+
+case "$old_vsn" in
+    e*)
+        cur_vsn="$(./pkg-vsn.sh emqx-enterprise)"
+        profile='emqx-enterprise'
+        ;;
+    v*)
+        cur_vsn="$(./pkg-vsn.sh emqx)"
+        profile='emqx'
+        ;;
+    *)
+        echo "unknown old version $old_vsn"
+        exit 1
+        ;;
+esac
+
+# From now on, no need for the v|e prefix
+OLD_VSN="${old_vsn#[e|v]}"
+
+OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
+CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
+
+if [ ! -f "$OLD_PKG" ]; then
+    echo "$OLD_PKG not found"
+    exit 1
+fi
+
+if [ ! -f "$CUR_PKG" ]; then
+    echo "$CUR_PKG not found"
+    exit 1
+fi
+
+# start two nodes and their friends (webhook server and a bench) in docker
+./scripts/relup/start-relup-test-cluster.sh 'ubuntu:20.04' "$OLD_PKG"
+
+# run relup tests
+lux \
+    --progress verbose \
+    --case_timeout infinity \
+    --var PROJ_ROOT="$(pwd)" \
+    --var VSN="$cur_vsn" \
+    --var CUR_PKG="$CUR_PKG" \
+    --var OLD_VSN="$OLD_VSN" \
+    --var NODE1="node1.emqx.io" \
+    --var NODE2="node2.emqx.io" \
+    --var BENCH="bench.emqx.io" \
+    ./scripts/relup/relup.lux
diff --git a/scripts/relup/start-relup-test-cluster.sh b/scripts/relup/start-relup-test-cluster.sh
new file mode 100755
index 000000000..deaec4ad1
--- /dev/null
+++ b/scripts/relup/start-relup-test-cluster.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+## EMQX can only start with longname (https://erlang.org/doc/reference_manual/distributed.html)
+## The host name part of EMQX's node name has to be static, this means we should either
+## pre-assign static IP for containers, or ensure containers can communiate with each other by name
+## this is why a docker network is created, and the containers's names have a dot.
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")/../.."
+
+set -x
+
+IMAGE="${1}"
+PKG="$(readlink -f "${2}")"
+
+NET='emqx.io'
+NODE1="node1.$NET"
+NODE2="node2.$NET"
+WEBHOOK="webhook.$NET"
+BENCH="bench.$NET"
+COOKIE='this-is-a-secret'
+## Erlang image is needed to run webhook server and emqtt-bench
+ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
+# builder has emqtt-bench installed
+BENCH_IMAGE="$ERLANG_IMAGE"
+
+## clean up
+docker rm -f "$BENCH" >/dev/null 2>&1 || true
+docker rm -f "$WEBHOOK" >/dev/null 2>&1 || true
+docker rm -f "$NODE1" >/dev/null 2>&1 || true
+docker rm -f "$NODE2" >/dev/null 2>&1 || true
+docker network rm "$NET" >/dev/null 2>&1 || true
+
+docker network create "$NET"
+
+docker run -d -t --name "$NODE1" \
+  --net "$NET" \
+  -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=warning \
+  -e EMQX_NODE_NAME="emqx@$NODE1" \
+  -e EMQX_NODE_COOKIE="$COOKIE" \
+  -p 18083:18083 \
+  -v "$PKG:/emqx.tar.gz" \
+  -v "$(pwd)/scripts/relup/run-pkg.sh:/run-pkg.sh" \
+  "$IMAGE" /run-pkg.sh emqx.tar.gz
+
+docker run -d -t --name "$NODE2" \
+  --net "$NET" \
+  -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=warning \
+  -e EMQX_NODE_NAME="emqx@$NODE2" \
+  -e EMQX_NODE_COOKIE="$COOKIE" \
+  -p 18084:18083 \
+  -v "$PKG:/emqx.tar.gz" \
+  -v "$(pwd)/scripts/relup/run-pkg.sh:/run-pkg.sh" \
+  "$IMAGE" /run-pkg.sh emqx.tar.gz
+
+docker run -d -t --name "$WEBHOOK" \
+  --net "$NET" \
+  -v "$(pwd)/.ci/fvt_tests/http_server:/http_server" \
+  -w /http_server \
+  -p 7077:7077 \
+  "$ERLANG_IMAGE" bash -c 'rebar3 compile; erl -pa _build/default/lib/*/ebin -eval "http_server:start()"'
+
+docker run -d -t --name "$BENCH" \
+    --net "$NET" \
+    "$BENCH_IMAGE" \
+    bash -c 'sleep 10000; exit 1'
+
+wait_limit=60
+wait_for_emqx() {
+    wait_sec=0
+    container="$1"
+    wait_limit="$2"
+    set +x
+    while ! docker exec "$container" emqx_ctl status >/dev/null 2>&1; do
+        wait_sec=$(( wait_sec + 1 ))
+        if [ $wait_sec -gt "$wait_limit" ]; then
+            echo "timeout wait for EMQX"
+            exit 1
+        fi
+        echo -n '.'
+        sleep 1
+    done
+}
+
+wait_for_webhook() {
+    wait_sec=0
+    wait_limit="$1"
+    set +x
+    while ! curl -f -s localhost:7077; do
+        wait_sec=$(( wait_sec + 1 ))
+        if [ $wait_sec -gt "$wait_limit" ]; then
+            echo "timeout wait for EMQX"
+            exit 1
+        fi
+        echo -n '.'
+        sleep 1
+    done
+}
+
+# wait for webhook http server to start,
+# it may take a while because it needs to compile from source code
+wait_for_webhook 120
+# after webhook start, it should not cost more than 30 seconds
+wait_for_emqx $NODE1 30
+# afer node1 is up, it should not cost more than 10 seconds
+wait_for_emqx $NODE2 10
+echo
+
+docker exec $NODE1 emqx_ctl cluster join "emqx@$NODE2"
diff --git a/scripts/run-relup-lux.sh b/scripts/run-relup-lux.sh
deleted file mode 100755
index 5b859c2c0..000000000
--- a/scripts/run-relup-lux.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-## This script needs the 'lux' command in PATH
-## it runs the .ci/fvt_tests/relup.lux script
-
-set -euo pipefail
-
-old_vsn="${1}"
-
-# ensure dir
-cd -P -- "$(dirname -- "$0")/.."
-
-set -x
-
-case "$old_vsn" in
-    e*)
-        cur_vsn="$(./pkg-vsn.sh emqx-enterprise)"
-        profile='emqx-enterprise'
-        ;;
-    v*)
-        cur_vsn="$(./pkg-vsn.sh emqx)"
-        profile='emqx'
-        ;;
-    *)
-        echo "unknown old version $old_vsn"
-        exit 1
-        ;;
-esac
-
-old_pkg="$(pwd)/_upgrade_base/${profile}-${old_vsn#[e|v]}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
-cur_pkg="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz"
-
-lux \
-    --progress verbose \
-    --case_timeout infinity \
-    --var PROJ_ROOT="$(pwd)" \
-    --var PROFILE="$profile" \
-    --var VSN="$cur_vsn" \
-    --var OLD_VSN="$old_vsn" \
-    --var CUR_PKG="$cur_pkg" \
-    --var OLD_PKG="$old_pkg" \
-    .ci/fvt_tests/relup.lux

From e16fdef6bf488bb7a861ff043035bebe0677ae6f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 08:00:55 +0200
Subject: [PATCH 18/23] ci: upload less data in build artifacts

---
 .github/workflows/run_relup_tests.yaml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 3d3223eb8..15ced040c 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -54,7 +54,10 @@ jobs:
       with:
         name: emqx_built
         path: |
-          emqx/*
+          emqx/_upgrade_base
+          emqx/_packages
+          emqx/scripts
+          emqx/.ci
 
   relup_test_run:
     needs:
@@ -98,7 +101,19 @@ jobs:
       run: |
         set -e -x -u
         cd emqx
-        if ! ./scripts/relup/run-relup-lux.sh $OLD_VSN; then
+        case "$OLD_VSN" in
+          e*)
+            export CUR_VSN="$CUR_EE_VSN"
+            ;;
+          v*)
+            export CUR_VSN="$CUR_CE_VSN"
+            ;;
+          *)
+            echo "unknown old version $OLD_VSN"
+            exit 1
+            ;;
+        esac
+        if ! ./scripts/relup-test/run-relup-lux.sh $OLD_VSN; then
           docker logs node1.emqx.io | tee lux_logs/emqx1.log
           docker logs node2.emqx.io | tee lux_logs/emqx2.log
           exit 1

From 4d4634335ced943e45b80d70c2fad0e27ad5fc67 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 08:01:05 +0200
Subject: [PATCH 19/23] ci: rename scripts/relup/ to scripts/relup-test

---
 scripts/relup-test/README.md                      | 10 ++++++++++
 scripts/{relup => relup-test}/check-results.sh    |  0
 scripts/{relup => relup-test}/relup.lux           |  2 +-
 scripts/{relup => relup-test}/run-pkg.sh          |  0
 scripts/{relup => relup-test}/run-relup-lux.sh    | 15 ++++++++++-----
 .../start-relup-test-cluster.sh                   |  4 ++--
 6 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 scripts/relup-test/README.md
 rename scripts/{relup => relup-test}/check-results.sh (100%)
 rename scripts/{relup => relup-test}/relup.lux (98%)
 rename scripts/{relup => relup-test}/run-pkg.sh (100%)
 rename scripts/{relup => relup-test}/run-relup-lux.sh (73%)
 rename scripts/{relup => relup-test}/start-relup-test-cluster.sh (96%)

diff --git a/scripts/relup-test/README.md b/scripts/relup-test/README.md
new file mode 100644
index 000000000..ff2db1ed9
--- /dev/null
+++ b/scripts/relup-test/README.md
@@ -0,0 +1,10 @@
+# Hot-upgrade test
+
+This collection of scripts is used in CI.
+
+It can also be used to run the test locally, but limited to ubuntu 20.04 so far.
+
+How to:
+```
+
+```
diff --git a/scripts/relup/check-results.sh b/scripts/relup-test/check-results.sh
similarity index 100%
rename from scripts/relup/check-results.sh
rename to scripts/relup-test/check-results.sh
diff --git a/scripts/relup/relup.lux b/scripts/relup-test/relup.lux
similarity index 98%
rename from scripts/relup/relup.lux
rename to scripts/relup-test/relup.lux
index 88a504cb8..8db8169f8 100644
--- a/scripts/relup/relup.lux
+++ b/scripts/relup-test/relup.lux
@@ -116,7 +116,7 @@
     !sleep 5
     ?SH-PROMPT
 
-    !$PROJ_ROOT/scripts/relup/check-results.sh
+    !$PROJ_ROOT/scripts/relup-test/check-results.sh
     !echo ==$$?==
     ???ALL_IS_WELL
     ?SH-PROMPT:
diff --git a/scripts/relup/run-pkg.sh b/scripts/relup-test/run-pkg.sh
similarity index 100%
rename from scripts/relup/run-pkg.sh
rename to scripts/relup-test/run-pkg.sh
diff --git a/scripts/relup/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh
similarity index 73%
rename from scripts/relup/run-relup-lux.sh
rename to scripts/relup-test/run-relup-lux.sh
index af0ecf4c7..54be86444 100755
--- a/scripts/relup/run-relup-lux.sh
+++ b/scripts/relup-test/run-relup-lux.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 ## This script needs the 'lux' command in PATH
-## it runs the scripts/relup/relup.lux script
+## it runs the scripts/relup-test/relup.lux script
 
 set -euo pipefail
 
@@ -16,13 +16,18 @@ cd -P -- "$(dirname -- "$0")/../.."
 
 set -x
 
+if [ ! -d '.git' ] && [ -z "${CUR_VSN:-}" ]; then
+    echo "Unable to resolve current version, because it's not a git repo, and CUR_VSN is not set"
+    exit 1
+fi
+
 case "$old_vsn" in
     e*)
-        cur_vsn="$(./pkg-vsn.sh emqx-enterprise)"
+        cur_vsn="${CUR_VSN:-$(./pkg-vsn.sh emqx-enterprise)}"
         profile='emqx-enterprise'
         ;;
     v*)
-        cur_vsn="$(./pkg-vsn.sh emqx)"
+        cur_vsn="${CUR_VSN:-$(./pkg-vsn.sh emqx)}"
         profile='emqx'
         ;;
     *)
@@ -48,7 +53,7 @@ if [ ! -f "$CUR_PKG" ]; then
 fi
 
 # start two nodes and their friends (webhook server and a bench) in docker
-./scripts/relup/start-relup-test-cluster.sh 'ubuntu:20.04' "$OLD_PKG"
+./scripts/relup-test/start-relup-test-cluster.sh 'ubuntu:20.04' "$OLD_PKG"
 
 # run relup tests
 lux \
@@ -61,4 +66,4 @@ lux \
     --var NODE1="node1.emqx.io" \
     --var NODE2="node2.emqx.io" \
     --var BENCH="bench.emqx.io" \
-    ./scripts/relup/relup.lux
+    ./scripts/relup-test/relup.lux
diff --git a/scripts/relup/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh
similarity index 96%
rename from scripts/relup/start-relup-test-cluster.sh
rename to scripts/relup-test/start-relup-test-cluster.sh
index deaec4ad1..10bb25a60 100755
--- a/scripts/relup/start-relup-test-cluster.sh
+++ b/scripts/relup-test/start-relup-test-cluster.sh
@@ -42,7 +42,7 @@ docker run -d -t --name "$NODE1" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -p 18083:18083 \
   -v "$PKG:/emqx.tar.gz" \
-  -v "$(pwd)/scripts/relup/run-pkg.sh:/run-pkg.sh" \
+  -v "$(pwd)/scripts/relup-test/run-pkg.sh:/run-pkg.sh" \
   "$IMAGE" /run-pkg.sh emqx.tar.gz
 
 docker run -d -t --name "$NODE2" \
@@ -52,7 +52,7 @@ docker run -d -t --name "$NODE2" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -p 18084:18083 \
   -v "$PKG:/emqx.tar.gz" \
-  -v "$(pwd)/scripts/relup/run-pkg.sh:/run-pkg.sh" \
+  -v "$(pwd)/scripts/relup-test/run-pkg.sh:/run-pkg.sh" \
   "$IMAGE" /run-pkg.sh emqx.tar.gz
 
 docker run -d -t --name "$WEBHOOK" \

From 2af629004008358f5bc9dd3aa785059b4ac840ef Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 08:28:57 +0200
Subject: [PATCH 20/23] ci: fix work dir

---
 .github/workflows/run_relup_tests.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index 15ced040c..a4d646343 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -100,7 +100,9 @@ jobs:
     - name: run relup test
       run: |
         set -e -x -u
-        cd emqx
+        chmod a+x scripts/**/*.sh
+        ls -l scripts
+        ls -l scripts/relup-test
         case "$OLD_VSN" in
           e*)
             export CUR_VSN="$CUR_EE_VSN"
@@ -113,6 +115,7 @@ jobs:
             exit 1
             ;;
         esac
+        mkdir -p lux_logs
         if ! ./scripts/relup-test/run-relup-lux.sh $OLD_VSN; then
           docker logs node1.emqx.io | tee lux_logs/emqx1.log
           docker logs node2.emqx.io | tee lux_logs/emqx2.log

From 888f637cdd2ba859b2bfac3b4203163cbc9f6f20 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 11:01:58 +0200
Subject: [PATCH 21/23] docs(ci): add how to run in
 scripts/relup-test/README.md

---
 scripts/relup-test/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/relup-test/README.md b/scripts/relup-test/README.md
index ff2db1ed9..a0a13f4fa 100644
--- a/scripts/relup-test/README.md
+++ b/scripts/relup-test/README.md
@@ -5,6 +5,7 @@ This collection of scripts is used in CI.
 It can also be used to run the test locally, but limited to ubuntu 20.04 so far.
 
 How to:
-```
 
 ```
+./scripts/relup-test/run-relup-lux.sh v5.0.0
+```

From 01e29fb82d70f5cf3786c3a79c00c6c6fc4684e5 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 12:38:27 +0200
Subject: [PATCH 22/23] ci(relup): delete TODO, check both nodes for metrics
 data

---
 scripts/relup-test/check-results.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/relup-test/check-results.sh b/scripts/relup-test/check-results.sh
index 446ebeca5..f6e20bda9 100755
--- a/scripts/relup-test/check-results.sh
+++ b/scripts/relup-test/check-results.sh
@@ -3,13 +3,9 @@
 set -euo pipefail
 
 matched_node1="$(curl --user admin:public -sf http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node1.emqx.io\") | .metrics.matched")"
-# TODO
-matched_node2=0
-#matched_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.matched")"
+matched_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.matched")"
 success_node1="$(curl --user admin:public -sf http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node1.emqx.io\") | .metrics.\"actions.success\"")"
-# TODO
-success_node2=0
-#success_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.\"actions.success\"")"
+success_node2="$(curl --user admin:public -sf http://localhost:18084/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@node2.emqx.io\") | .metrics.\"actions.success\"")"
 webhook="$(curl -sf http://localhost:7077/counter | jq '.data')"
 
 MATCHED_TOTAL="$(( matched_node1 + matched_node2 ))"

From 84d2fde9875d054d4e5c3e246dce04046e4b8a19 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 29 Jun 2022 14:41:16 +0200
Subject: [PATCH 23/23] ci: fix relup log upload path (in case of failure)

Co-authored-by: Thales Macedo Garitezi <thalesmg@gmail.com>
---
 .github/workflows/run_relup_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index a4d646343..ddb8380b2 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -127,4 +127,4 @@ jobs:
       with:
         name: debug_data
         path: |
-          wd/lux_logs
+          lux_logs