From 5d81f159d6834229ad00dedb433e9ddcefcc6e87 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Jun 2022 14:00:27 +0800 Subject: [PATCH 1/4] fix(CI): syntax errors in run_relup_tests.yaml --- .ci/fvt_tests/relup.lux | 2 +- .github/workflows/run_relup_tests.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index f5b47b89e..9e67dfd63 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -22,7 +22,7 @@ [shell emqx] !cd $PACKAGE_PATH - mkdir -p emqx + !mkdir -p emqx !tar -C emqx -zxf ${PROFILE}-$(echo $old_vsn | sed -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz ?SH-PROMPT diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 97c33f229..9783d0781 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -70,8 +70,9 @@ jobs: fetch-depth: 0 - name: Get old vsn - run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV - run: echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV + run: | + echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV + echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - name: build emqx env: From efffc450d0cf148b7b3d891da30676945fff6923 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Jun 2022 14:00:27 +0800 Subject: [PATCH 2/4] fix: update relup.lux --- .ci/fvt_tests/http_server/src/http_server.erl | 55 ++++++++++--------- .ci/fvt_tests/relup.lux | 47 +++++++--------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl index b66b72939..4b1981de4 100644 --- a/.ci/fvt_tests/http_server/src/http_server.erl +++ b/.ci/fvt_tests/http_server/src/http_server.erl @@ -1,36 +1,41 @@ -module(http_server). --import(minirest, [ return/0 - , return/1 - ]). +-import(minirest, [ + return/0, + return/1 +]). --export([ start/0 - , stop/0 - ]). +-export([ + start/0, + stop/0 +]). --rest_api(#{ name => get_counter - , method => 'GET' - , path => "/counter" - , func => get_counter - , descr => "Check counter" - }). --rest_api(#{ name => add_counter - , method => 'POST' - , path => "/counter" - , func => add_counter - , descr => "Counter plus one" - }). +-rest_api(#{ + name => get_counter, + method => 'GET', + path => "/counter", + func => get_counter, + descr => "Check counter" +}). +-rest_api(#{ + name => add_counter, + method => 'POST', + path => "/counter", + func => add_counter, + descr => "Counter plus one" +}). --export([ get_counter/2 - , add_counter/2 - ]). +-export([ + get_counter/2, + add_counter/2 +]). start() -> application:ensure_all_started(minirest), ets:new(relup_test_message, [named_table, public]), Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}], Dispatch = [{"/[...]", minirest, Handlers}], - minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 8080}]}, Dispatch). + minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 7077}]}, Dispatch). stop() -> ets:delete(relup_test_message), @@ -39,12 +44,12 @@ stop() -> get_counter(_Binding, _Params) -> return({ok, ets:info(relup_test_message, size)}). -add_counter(_Binding, Params) -> +add_counter(_Binding, Params) -> case lists:keymember(<<"payload">>, 1, Params) of - true -> + true -> {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params), ets:insert(relup_test_message, {ID, Params1}); - _ -> + _ -> ok end, return(). diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 9e67dfd63..a6de27e94 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -16,7 +16,7 @@ ???Eshell ???> !http_server:start(). - ?Start http_server listener on 8080 successfully. + ?Start http_server listener on 7077 successfully. ?ok ?> @@ -27,9 +27,6 @@ ?SH-PROMPT !cd emqx - !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf - !sed -i '/emqx_telemetry/d' data/loaded_plugins - !./bin/emqx start ?EMQ X .* is started successfully! ?SH-PROMPT @@ -53,24 +50,27 @@ !./bin/emqx_ctl cluster status """??? - Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], - stopped_nodes => []} + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []} """ ?SH-PROMPT - !./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}' - ?created + ## create a webhook data bridge with id "my_webhook" + !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","method":"post","url":"http://127.0.0.1:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status' + ?connected ?SH-PROMPT - !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource": "resource:691c29ba"}}]' - ?created + + ## create a rule that uses the webhook as action, the rule id = "rule_edsy" + !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"id":"rule_edsy","sql":"SELECT\n *\nFROM\n \"t/#\"","actions":["webhook:my_webhook"]}' | jq '.id' + ?rule_edsy ?SH-PROMPT [shell emqx] - !./bin/emqx_ctl resources list - ?691c29ba + ## verify the bridges and rules are sync to the other node + !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges/webhook:my_webhook' -X 'GET' -H 'Content-Type: application/json' | jq '.name' + ?my_webhook ?SH-PROMPT - !./bin/emqx_ctl rules list - ?691c29ba + !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/rules/rule_edsy' -X 'GET' -H 'Content-Type: application/json' | jq '.id' + ?rule_edsy ?SH-PROMPT [shell bench] @@ -116,15 +116,10 @@ !./bin/emqx_ctl cluster status """??? - Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], - stopped_nodes => []} + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []} """ ?SH-PROMPT - !./bin/emqx_ctl plugins list | grep emqx_management - ?Plugin\(emqx_management.*active=true\) - ?SH-PROMPT - [shell emqx2] !echo "" > log/emqx.log.1 ?SH-PROMPT @@ -162,15 +157,10 @@ !./bin/emqx_ctl cluster status """??? - Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], - stopped_nodes => []} + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],stopped_nodes => []} """ ?SH-PROMPT - !./bin/emqx_ctl plugins list | grep emqx_management - ?Plugin\(emqx_management.*active=true\) - ?SH-PROMPT - ## We don't guarantee not to lose a single message! ## So even if we received 290~300 messages, we consider it as success [shell bench] @@ -187,6 +177,11 @@ ?\{"data":(29[0-9])|(300),"code":0\} ?SH-PROMPT + ## The /counter API is provided by .ci/fvt_test/http_server + !curl http://127.0.0.1:7077/counter + ?\{"data":(29[0-9])|(300),"code":0\} + ?SH-PROMPT + [shell emqx2] !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100 -error From 0d980904e09e0dd21a890cd7ae6ad3e7250ce563 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Jun 2022 22:16:03 +0800 Subject: [PATCH 3/4] fix: tmp - test relup --- .ci/fvt_tests/relup.lux | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index a6de27e94..f3ac60521 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -27,7 +27,7 @@ ?SH-PROMPT !cd emqx - !./bin/emqx start + !./bin/emqx console ?EMQ X .* is started successfully! ?SH-PROMPT @@ -40,7 +40,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins - !./bin/emqx start + !./bin/emqx console ?EMQ X .* is started successfully! ?SH-PROMPT From 9ec88198cbe54e4d0a0959335b3b291ea64d1e81 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 23 Jun 2022 09:14:57 +0800 Subject: [PATCH 4/4] fix: remove one_more_emqx.sh from the relup.lux --- .ci/fvt_tests/relup.lux | 17 +++++++++++------ .github/workflows/run_relup_tests.yaml | 10 +++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index f3ac60521..1972d33ee 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -1,7 +1,6 @@ [config var=PROFILE] [config var=PACKAGE_PATH] [config var=BENCH_PATH] -[config var=ONE_MORE_EMQX_PATH] [config var=VSN] [config var=OLD_VSNS] @@ -33,14 +32,20 @@ [shell emqx2] !cd $PACKAGE_PATH - !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh . - !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2 + !cp -r emqx emqx2 ?SH-PROMPT !cd emqx2 - !sed -i '/emqx_telemetry/d' data/loaded_plugins - - !./bin/emqx console + """! + EMQX_NODE__NAME='emqx2@127.0.0.1' \ + EMQX_STATSD__SERVER='127.0.0.1:8124' \ + EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1882' \ + EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8882' \ + EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8082' \ + EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8085' \ + EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18082' \ + ./bin/emqx console + """ ?EMQ X .* is started successfully! ?SH-PROMPT diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 9783d0781..628684089 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,6 +15,8 @@ on: jobs: relup_test: + runs-on: aws-amd64 + strategy: matrix: profile: @@ -32,13 +34,13 @@ jobs: arch: - amd64 - runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: shell: bash steps: + - uses: AutoModality/action-clean@v1 - uses: actions/setup-python@v2 with: python-version: '3.8' @@ -48,11 +50,6 @@ jobs: repository: emqx/paho.mqtt.testing ref: develop-4.0 path: paho.mqtt.testing - - uses: actions/checkout@v2 - with: - repository: terry-xiaoyu/one_more_emqx - ref: master - path: one_more_emqx - uses: actions/checkout@v2 with: repository: emqx/emqtt-bench @@ -103,7 +100,6 @@ jobs: --var PROFILE=$PROFILE \ --var PACKAGE_PATH=$(pwd)/packages \ --var BENCH_PATH=$(pwd)/emqtt-bench \ - --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSNS="$OLD_VSNS" \ emqx/.ci/fvt_tests/relup.lux